diff --git a/.config/jp/tools/Cargo.toml b/.config/jp/tools/Cargo.toml index 0649f20a9..433da5eb1 100644 --- a/.config/jp/tools/Cargo.toml +++ b/.config/jp/tools/Cargo.toml @@ -29,8 +29,8 @@ crossbeam-channel = { workspace = true, features = ["std"] } crossterm = { workspace = true } duct = { workspace = true } fancy-regex = { workspace = true, features = ["perf", "std", "unicode"] } +grep-matcher = { workspace = true } grep-printer = { workspace = true } -grep-regex = { workspace = true } grep-searcher = { workspace = true } htmd = { workspace = true } ignore = { workspace = true } diff --git a/.config/jp/tools/src/fs/grep_files.rs b/.config/jp/tools/src/fs/grep_files.rs index 2f58c6d48..ba4fff7d7 100644 --- a/.config/jp/tools/src/fs/grep_files.rs +++ b/.config/jp/tools/src/fs/grep_files.rs @@ -1,13 +1,15 @@ use camino::{Utf8Path, Utf8PathBuf}; use grep_printer::StandardBuilder; -use grep_regex::RegexMatcher; use grep_searcher::{BinaryDetection, SearcherBuilder}; use ignore::gitignore::Gitignore; use jp_tool::AccessPolicy; +use matcher::FancyMatcher; use super::fs_list_files; use crate::{Error, util::OneOrMany}; +mod matcher; + pub(crate) async fn fs_grep_files( root: &Utf8Path, access: Option<&AccessPolicy>, @@ -45,7 +47,7 @@ pub(crate) async fn fs_grep_files( pattern = format!("{pattern}|{pat}"); } - let matcher = RegexMatcher::new(&pattern)?; + let matcher = FancyMatcher::new(&pattern)?; let mut printer = StandardBuilder::new() .max_columns(Some(1000)) diff --git a/.config/jp/tools/src/fs/grep_files/matcher.rs b/.config/jp/tools/src/fs/grep_files/matcher.rs new file mode 100644 index 000000000..6e6330fc1 --- /dev/null +++ b/.config/jp/tools/src/fs/grep_files/matcher.rs @@ -0,0 +1,54 @@ +use fancy_regex::{Regex, RegexBuilder}; +use grep_matcher::{Match, Matcher, NoCaptures}; + +/// A search matcher backed by `fancy-regex`. +/// +/// Patterns get the same dialect as the rest of the toolset, so lookaround +/// (`(?=...)`, `(? Result { + // Line-oriented anchoring: `^` and `$` bind to line boundaries rather + // than to the ends of the haystack, because the searcher hands over + // whole buffers and callers write grep patterns, not whole-file ones. + let regex = RegexBuilder::new(pattern) + .multi_line(true) + .dot_matches_new_line(false) + .unicode_mode(true) + .build()?; + + Ok(Self { regex }) + } +} + +impl Matcher for FancyMatcher { + type Captures = NoCaptures; + type Error = fancy_regex::Error; + + fn find_at(&self, haystack: &[u8], at: usize) -> Result, Self::Error> { + // `find_from_pos` starts the search at `at` while leaving the preceding + // bytes visible, which is what lets lookbehind work across the offset. + // Slicing the haystack instead would hide them. + // + // Bytes that are not valid UTF-8 simply never match, so one latin-1 + // line does not blind the search to the rest of the buffer. + let found = self + .regex + .find_from_pos(haystack, at)? + .map(|m| Match::new(m.start(), m.end())); + + Ok(found) + } + + fn new_captures(&self) -> Result { + Ok(NoCaptures::new()) + } +} + +#[cfg(test)] +#[path = "matcher_tests.rs"] +mod tests; diff --git a/.config/jp/tools/src/fs/grep_files/matcher_tests.rs b/.config/jp/tools/src/fs/grep_files/matcher_tests.rs new file mode 100644 index 000000000..184d7f2e5 --- /dev/null +++ b/.config/jp/tools/src/fs/grep_files/matcher_tests.rs @@ -0,0 +1,48 @@ +use super::*; + +#[test] +fn reports_absolute_offsets_when_searching_from_a_position() { + let matcher = FancyMatcher::new("needle").unwrap(); + + let found = matcher.find_at(b"needle and needle", 7).unwrap().unwrap(); + + assert_eq!((found.start(), found.end()), (11, 17)); +} + +#[test] +fn lookbehind_reaches_behind_the_start_position() { + let matcher = FancyMatcher::new("(?<=let )needle").unwrap(); + + // The match begins at 4, so a search from 4 only succeeds if the bytes + // before it are still visible to the lookbehind. + let found = matcher.find_at(b"let needle", 4).unwrap().unwrap(); + + assert_eq!((found.start(), found.end()), (4, 10)); +} + +#[test] +fn caret_anchors_to_a_line_start_within_the_buffer() { + let matcher = FancyMatcher::new("^beta").unwrap(); + + let found = matcher.find_at(b"alpha\nbeta\n", 0).unwrap().unwrap(); + + assert_eq!((found.start(), found.end()), (6, 10)); +} + +#[test] +fn offsets_stay_absolute_in_a_haystack_holding_invalid_utf8() { + let matcher = FancyMatcher::new("needle").unwrap(); + + // The latin-1 byte occupies one byte, so the match starts at 5. Reporting + // it anywhere else would make the searcher slice the wrong line. + let found = matcher.find_at(b"caf\xe9 needle", 0).unwrap().unwrap(); + + assert_eq!((found.start(), found.end()), (5, 11)); +} + +#[test] +fn a_pattern_that_cannot_match_returns_none_rather_than_an_error() { + let matcher = FancyMatcher::new("needle").unwrap(); + + assert!(matcher.find_at(b"nothing here", 0).unwrap().is_none()); +} diff --git a/.config/jp/tools/src/fs/grep_files_tests.rs b/.config/jp/tools/src/fs/grep_files_tests.rs index ba0a04863..aca3cd92e 100644 --- a/.config/jp/tools/src/fs/grep_files_tests.rs +++ b/.config/jp/tools/src/fs/grep_files_tests.rs @@ -319,6 +319,97 @@ async fn suppressed_path_is_reported_so_the_caller_can_ask_the_user() { ); } +/// Search a single-file workspace, returning the tool's rendered output. +/// +/// The pattern semantics below are the contract the tool exposes to callers; +/// they must hold for whichever regex engine backs the search. +async fn grep_one_file(content: &[u8], pattern: &str) -> String { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("a.txt"), content).unwrap(); + + fs_grep_files( + root, + None, + pattern.to_owned(), + None, + None, + None, + &Gitignore::empty(), + ) + .await + .unwrap() +} + +#[tokio::test] +async fn caret_and_dollar_anchor_per_line() { + let matches = grep_one_file(b"alpha\nbeta\nalphabet\n", "^alpha$").await; + + assert_eq!(matches, "a.txt:1:alpha\n"); +} + +#[tokio::test] +async fn dot_does_not_match_across_lines() { + let matches = grep_one_file(b"a\nb\n", "a.b").await; + + assert_eq!( + matches, + "No matches found. Broaden your search to see more." + ); +} + +#[tokio::test] +async fn word_boundaries_hold_and_matching_is_case_sensitive() { + let matches = grep_one_file(b"foo\nfoobar\nFOO\n", r"\bfoo\b").await; + + assert_eq!(matches, "a.txt:1:foo\n"); +} + +#[tokio::test] +async fn repeated_match_on_one_line_prints_the_line_once() { + let matches = grep_one_file(b"foo bar foo\n", "foo").await; + + assert_eq!(matches, "a.txt:1:foo bar foo\n"); +} + +#[tokio::test] +async fn a_zero_width_pattern_matches_an_empty_line() { + let matches = grep_one_file(b"alpha\n\nbeta\n", "^$").await; + + assert_eq!(matches, "a.txt:2:\n"); +} + +#[tokio::test] +async fn a_line_that_is_not_utf8_does_not_abort_the_search() { + // Latin-1 bytes in a text file are not binary (no NUL), so the search runs + // on. Only the matched line reaches the output, which keeps the final + // UTF-8 decode of the printer buffer intact. + let matches = grep_one_file(b"caf\xe9 latte\nneedle here\n", "needle").await; + + assert_eq!(matches, "a.txt:2:needle here\n"); +} + +#[tokio::test] +async fn negative_lookahead_excludes_a_match() { + let matches = grep_one_file(b"foo bar\nfoo baz\n", "foo (?!bar)").await; + + assert_eq!(matches, "a.txt:2:foo baz\n"); +} + +#[tokio::test] +async fn lookbehind_sees_text_before_the_match() { + let matches = grep_one_file(b"let needle\nfn needle\n", "(?<=let )needle").await; + + assert_eq!(matches, "a.txt:1:let needle\n"); +} + +#[tokio::test] +async fn backreference_matches_a_repeated_group() { + let matches = grep_one_file(b"hello hello\nhello world\n", r"(\w+) \1").await; + + assert_eq!(matches, "a.txt:1:hello hello\n"); +} + #[tokio::test] #[test_log::test] async fn test_grep_files() { diff --git a/.config/supply-chain/audits.toml b/.config/supply-chain/audits.toml index 238bf7576..499a56485 100644 --- a/.config/supply-chain/audits.toml +++ b/.config/supply-chain/audits.toml @@ -61,6 +61,11 @@ who = "Jean Mertz " criteria = "safe-to-deploy" delta = "0.16.2 -> 0.17.0" +[[audits.fancy-regex]] +who = "Jean Mertz " +criteria = "safe-to-deploy" +delta = "0.17.0 -> 0.19.0" + [[audits.finl_unicode]] who = "Jean Mertz " criteria = "safe-to-deploy" diff --git a/.config/supply-chain/imports.lock b/.config/supply-chain/imports.lock index bf20c5af1..d30398cdd 100644 --- a/.config/supply-chain/imports.lock +++ b/.config/supply-chain/imports.lock @@ -261,13 +261,6 @@ user-id = 189 user-login = "BurntSushi" user-name = "Andrew Gallant" -[[publisher.grep-regex]] -version = "0.1.13" -when = "2024-09-09" -user-id = 189 -user-login = "BurntSushi" -user-name = "Andrew Gallant" - [[publisher.grep-searcher]] version = "0.1.16" when = "2025-10-22" @@ -581,8 +574,8 @@ user-login = "BurntSushi" user-name = "Andrew Gallant" [[publisher.regex-automata]] -version = "0.4.10" -when = "2025-08-24" +version = "0.4.18" +when = "2026-08-04" user-id = 189 user-login = "BurntSushi" user-name = "Andrew Gallant" diff --git a/.jp/mcp/tools/fs/grep_files.toml b/.jp/mcp/tools/fs/grep_files.toml index ff1422bd6..56447cd4d 100644 --- a/.jp/mcp/tools/fs/grep_files.toml +++ b/.jp/mcp/tools/fs/grep_files.toml @@ -54,7 +54,7 @@ parameters = "function_call" [conversation.tools.fs_grep_files.parameters.pattern] type = "string" required = true -summary = "Regular expression to filter the results by." +summary = "Regular expression to filter the results by. Lookaround and backreferences are supported." [conversation.tools.fs_grep_files.parameters.context] type = "integer" diff --git a/.jp/mcp/tools/fs/grep_user_docs.toml b/.jp/mcp/tools/fs/grep_user_docs.toml index c55117e5c..a860d7fa1 100644 --- a/.jp/mcp/tools/fs/grep_user_docs.toml +++ b/.jp/mcp/tools/fs/grep_user_docs.toml @@ -39,7 +39,7 @@ parameters = "function_call" [conversation.tools.fs_grep_user_docs.parameters.pattern] type = "string" required = true -summary = "Regular expression to filter the results by." +summary = "Regular expression to filter the results by. Lookaround and backreferences are supported." [conversation.tools.fs_grep_user_docs.parameters.context] type = "integer" diff --git a/.jp/mcp/tools/fs/modify_file.toml b/.jp/mcp/tools/fs/modify_file.toml index 4a14f93e1..b329f00d9 100644 --- a/.jp/mcp/tools/fs/modify_file.toml +++ b/.jp/mcp/tools/fs/modify_file.toml @@ -102,7 +102,7 @@ properties.regex = { type = "boolean", description = "Whether to treat this patt required = true default = false type = "boolean" -summary = "Whether to treat `old` as a regular expression." +summary = "Whether to treat `old` as a regular expression. Lookaround and backreferences are supported." description = """ If `true`, `old` is treated as a regular expression and the `new` is treated \ as a replacement string, which may contain capture groups. @@ -112,6 +112,9 @@ a literal string. Individual patterns can override this via their own `regex` field. +Patterns are compiled with `fancy-regex`, so lookahead (`(?=...)`, `(?!...)`), \ +lookbehind (`(?<=...)`, `(? gimli = { version = "0.33" } glob = { version = "0.3", default-features = false } +grep-matcher = { version = "0.1", default-features = false } grep-printer = { version = "0.3", default-features = false } -grep-regex = { version = "0.1", default-features = false } grep-searcher = { version = "0.1", default-features = false } htmd = { version = "0.5", default-features = false } httpmock = { git = "https://github.com/JeanMertz/httpmock", branch = "tweaks", default-features = false } diff --git a/crates/jp_cli/src/cmd/conversation/grep_tests.rs b/crates/jp_cli/src/cmd/conversation/grep_tests.rs index a0c5aff06..32d0ecb81 100644 --- a/crates/jp_cli/src/cmd/conversation/grep_tests.rs +++ b/crates/jp_cli/src/cmd/conversation/grep_tests.rs @@ -1037,6 +1037,18 @@ fn quiet_reports_a_match_through_its_exit_status_alone() { ); } +/// A line that drives the pattern `(a+)+\1b` past its backtrack limit. +/// +/// The leading `b` is the part that matters. +/// With the pattern's required literal absent from the line, the search is +/// decided before the backtracking VM is ever entered, and a bare run of `a`s +/// is scanned cheaply. +/// Here the `b` is present but unreachable, since no match can end at it, so +/// the VM runs and exhausts its step budget. +fn backtracking_line() -> String { + format!("b{}", "a".repeat(4000)) +} + #[test] fn quiet_exits_zero_when_a_match_survives_a_failing_pattern() { // `grep -q`'s rule: if a line is selected the status is 0 even if an error @@ -1052,7 +1064,7 @@ fn quiet_exits_zero_when_a_match_survives_a_failing_pattern() { let (mut ctx, _out) = setup(vec![( id, turn(vec![ConversationEvent::new( - ChatRequest::from(format!("{}\naab", "a".repeat(64)).as_str()), + ChatRequest::from(format!("{}\naab", backtracking_line()).as_str()), ts(), )]), )]); @@ -1076,7 +1088,7 @@ fn quiet_exits_two_when_a_failure_leaves_no_match() { let (mut ctx, _out) = setup(vec![( id, turn(vec![ConversationEvent::new( - ChatRequest::from("a".repeat(64).as_str()), + ChatRequest::from(backtracking_line().as_str()), ts(), )]), )]); @@ -1159,7 +1171,7 @@ fn a_pattern_that_fails_mid_search_exits_two() { let (mut ctx, out) = setup(vec![( id, turn(vec![ConversationEvent::new( - ChatRequest::from("a".repeat(64).as_str()), + ChatRequest::from(backtracking_line().as_str()), ts(), )]), )]); diff --git a/crates/jp_cli/src/shared/search_tests.rs b/crates/jp_cli/src/shared/search_tests.rs index 483d6892e..3c685a6ad 100644 --- a/crates/jp_cli/src/shared/search_tests.rs +++ b/crates/jp_cli/src/shared/search_tests.rs @@ -121,8 +121,11 @@ fn a_poisoned_matcher_stops_matching() { assert!(matcher.failure().is_none()); // Nested quantifiers over a long same-character run blow the backtrack - // limit and poison the matcher. - assert!(!matcher.is_match(&"a".repeat(64))); + // limit and poison the matcher. The leading `b` is what puts the engine on + // that path at all: with the pattern's required literal absent from the + // line, the search is decided without ever entering the backtracking VM. + // Here the `b` is present but unreachable, since no match can end at it. + assert!(!matcher.is_match(&format!("b{}", "a".repeat(4000)))); assert!(matcher.failure().is_some()); assert!(