diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 6d05ca19f..8aa8e5bad 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -230,13 +230,25 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, prepare single term return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_tsquery_term(word: str) -> str: + """Render one relaxed word as a tsquery-safe prefix expression. + + Mirrors the SQLite renderer: a word token can contain an apostrophe, and + tsquery reads that as lexeme-quoting syntax rather than text. Quoting the + lexeme and doubling any interior quote keeps it literal. + """ + if "'" in word: + return "'{}':*".format(word.replace("'", "''")) + return f"{word}:*" + @staticmethod def _relaxed_tsquery_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed tsquery expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " | ".join(f"{word}:*" for word in words) + return " | ".join(PostgresSearchRepository._relaxed_tsquery_term(word) for word in words) def _prepare_boolean_query(self, query: str) -> str: """Convert Boolean query to tsquery format. diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 8544bdb47..0f5e10148 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -1,6 +1,7 @@ """Shared full-text query preparation rules.""" import re +import unicodedata # Interrogative/function words contribute lexical noise when a strict # full-text query is relaxed: "when OR did OR a" matches loud wrong documents @@ -25,8 +26,71 @@ r"\uff65-\uff9f" # Halfwidth Katakana r"]" ) -RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+") RELAXATION_EDGE_PUNCTUATION = "?!.,;:,。!?;:、" +# Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. +RELAXATION_JOIN_CONTROLS = "\u200c\u200d" +# Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", +# where the digit must stay its own token so the numeric guard still sees it. +RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" + + +def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: + """Whether a non-alphanumeric character belongs to the word being read. + + Combining marks, zero-width join controls, and apostrophes are written inside + a word but are not alphanumeric, so a naive scan treats them as separators + and splits one orthographic word into several tokens. + + An apostrophe counts only between two letters. That keeps "\u043f\u2019\u044f\u0442\u044c" whole while + leaving "SPEC 16's" split, so the digit stays a token of its own and the + numeric-identifier guard still rejects the query. + """ + char = text[index] + if char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M"): + return True + if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: + follows_letter = bool(current) and current[-1].isalpha() + precedes_letter = index + 1 < len(text) and text[index + 1].isalpha() + return follows_letter and precedes_letter + return False + + +def relaxation_word_tokens(text: str) -> list[str]: + """Split text into word tokens for the relaxation eligibility guards. + + A token is a run of alphanumeric characters together with the combining + marks, join controls, and apostrophes written inside it. Counting this way matters because + an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, + Armenian, and Georgian queries, so the three-token guard below rejected every + one of them and the hybrid FTS branch silently contributed nothing. + + Counting characters that live inside a word as separators is just as wrong in + the other direction: it cuts abugidas (Devanagari, Thai), decomposed text, + and Persian or Indic words joined by U+200C/U+200D into fragments. One word + then looks like several tokens, clears the three-token guard, and relaxes + into a broad OR of fragments — the opposite of what the guard is for. + """ + tokens: list[str] = [] + current: list[str] = [] + + def flush() -> None: + # Trailing join controls are word-internal by definition, so a token that + # ends in one is really a word followed by a separator. + token = "".join(current).rstrip(RELAXATION_JOIN_CONTROLS) + if token: + tokens.append(token) + current.clear() + + for index, char in enumerate(text): + # A leading mark, join control, or apostrophe has no base character to + # attach to, so it cannot open a token; that keeps stray punctuation from + # forming fragment-only terms. + if char.isalnum() or (current and _is_token_continuation(text, index, current)): + current.append(char) + elif current: + flush() + flush() + return tokens def _dedupe_relaxation_words(words: list[str]) -> list[str]: @@ -57,13 +121,16 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: - empty / quoted / explicit-boolean queries (user intent is not second-guessed); - - fewer than three alphanumeric tokens (short queries like "New Feature" + - fewer than three word tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize - to 1.0 and can outrank the vector result the user wanted); + to 1.0 and can outrank the vector result the user wanted). Tokens are + counted with relaxation_word_tokens, so scripts other than Latin reach the + same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because - the ASCII token gate would otherwise suppress the fallback entirely; - - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries - over-broaden and create false positives under OR. + they are not whitespace-delimited the way the token guard assumes; + - any numeric token ("root note 1", "SPEC 16", "SPEC \u216b") — identifier-like + queries over-broaden and create false positives under OR. Numeric-ness is + Unicode-wide, so Nl/No characters such as \u216b and \u00bd are caught too. """ if not search_text: return None @@ -77,7 +144,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) if has_cjk_term: - if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): + if len(cjk_words) < 2 or any(word.isnumeric() for word in cjk_words): return None pruned_words = [ word @@ -91,8 +158,8 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: # Outcome: preserve the short-query guard after pruning to avoid a broad retry. return relaxed_words if len(relaxed_words) >= 2 else None - tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower()) - if len(tokens) < 3 or any(token.isdigit() for token in tokens): + tokens = relaxation_word_tokens(stripped.lower()) + if len(tokens) < 3 or any(token.isnumeric() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] return _dedupe_relaxation_words(pruned_words or tokens) or None diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 91bcae435..9dced81f9 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -397,13 +397,26 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, use the single term preparation logic return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_fts_term(word: str) -> str: + """Render one relaxed word as an FTS5-safe prefix expression. + + A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated + bare it is FTS5 syntax, not text: the whole expression fails to parse, the + caller swallows the syntax error, and the relaxed retry returns nothing — + the exact silent-empty-FTS failure this fallback exists to prevent. + """ + if "'" in word or '"' in word: + return '"{}"*'.format(word.replace('"', '""')) + return f"{word}*" + @staticmethod def _relaxed_fts_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed FTS5 expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " OR ".join(f"{word}*" for word in words) + return " OR ".join(SQLiteSearchRepository._relaxed_fts_term(word) for word in words) @override async def semantic_effectively_enabled(self) -> bool: diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9a7181d8b..a5d9832a4 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -34,3 +34,154 @@ def test_relaxed_query_words_supports_whitespace_separated_cjk_scripts( def test_relaxed_query_words_preserves_short_query_guard_after_cjk_pruning(query: str) -> None: """Unsafe, duplicate, or stopword terms cannot pad a one-term CJK relaxation.""" assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("как отозвать выданный доступ", ["как", "отозвать", "выданный", "доступ"]), + ("як відкликати виданий доступ", ["як", "відкликати", "виданий", "доступ"]), + ("πώς να ανακαλέσετε πρόσβαση", ["πώς", "να", "ανακαλέσετε", "πρόσβαση"]), + ("כיצד לבטל גישה שניתנה", ["כיצד", "לבטל", "גישה", "שניתנה"]), + ("كيف تلغي الوصول الممنوح", ["كيف", "تلغي", "الوصول", "الممنوح"]), + ("ինչպես չեղարկել տրված մուտքը", ["ինչպես", "չեղարկել", "տրված", "մուտքը"]), + ], +) +def test_relaxed_query_words_supports_non_latin_alphabetic_scripts( + query: str, + expected: list[str], +) -> None: + """Non-Latin alphabetic queries reach the same guard as Latin ones. + + An ASCII-only token pattern found zero tokens in these queries, so the + three-token guard rejected every one of them and the hybrid FTS branch + contributed nothing — hybrid search silently became vector-only. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("पहुंच कैसे रद्द करें", ["पहुंच", "कैसे", "रद्द", "करें"]), # Devanagari + ("วิธี เพิกถอน การเข้าถึง", ["วิธี", "เพิกถอน", "การเข้าถึง"]), # Thai + ("como revogar acesso concedido", ["como", "revogar", "acesso", "concedido"]), + ], +) +def test_relaxed_query_words_keeps_combining_marks_with_their_base_character( + query: str, + expected: list[str], +) -> None: + """Vowel signs and diacritics stay inside the word they attach to. + + Combining marks are not alphanumeric, so treating them as separators splits + one abugida word into syllable fragments. The token count then inflates past + the three-token guard and relaxation ORs those fragments together. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "अंतर्राष्ट्रीयकरण", # one Devanagari word: 7 fragments if marks split it + "การเข้าถึง", # one Thai word + "pre\u0301sentation", # one word, NFD-decomposed acute accent + ], +) +def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str) -> None: + """A single word stays one token, so the short-query guard still rejects it.""" + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + "query", + [ + "отозвать доступ", # fewer than three tokens + "спека 16 доступ", # pure-digit token + '"точная фраза"', # quoted: user intent is explicit + "доступ OR токен", # explicit boolean: user intent is explicit + ], +) +def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None: + """Non-Latin queries gain no exemption from the short-query and identifier guards.""" + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("می‌روم خانه", None), # two Persian words, one joined by ZWNJ + ("نمی‌خواهم دسترسی را لغو", ["نمی‌خواهم", "دسترسی", "را", "لغو"]), + ("क‍ष विशेष पहुंच", ["क‍ष", "विशेष", "पहुंच"]), # explicit ZWJ conjunct + ], +) +def test_relaxed_query_words_keeps_join_controls_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """U+200C/U+200D are written inside a word, so they must not split its token. + + Splitting on them inflates the token count: a two-word Persian query looks + like three tokens, clears the three-token guard, and relaxes into fragments. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "SPEC Ⅻ design", # Nl: Roman numeral twelve + "spec ½ design", # No: vulgar fraction one half + "٣ ٤ ٥", # Arabic-Indic digits + ], +) +def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None: + """The identifier guard classifies numbers Unicode-wide, not just as ASCII digits. + + `isdigit()` is false for Nl/No characters, so admitting every alphanumeric + character would let identifier-like queries slip past the numeric guard. + """ + assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("п’ять проектів", None), # two Ukrainian words, U+2019 + ("об'єкт доступу", None), # two Ukrainian words, ASCII apostrophe + ( + "скасувати п’ять виданих об'єктів", + ["скасувати", "п’ять", "виданих", "об'єктів"], + ), + ], +) +def test_relaxed_query_words_keeps_apostrophes_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """A word-internal apostrophe must not split one word into several tokens. + + Splitting on it turned a two-word Ukrainian query into three tokens, which + cleared the three-token guard and relaxed into one-letter fragments. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_apostrophe_does_not_shield_numeric_tokens() -> None: + """An apostrophe joins letters only, so a digit stays a token of its own. + + Were `16's` read as one token it would not be numeric, and the query would + escape the identifier guard that rejects `SPEC 16 design`. + """ + assert relaxed_query_words("SPEC 16's design") is None + + +def test_relaxed_query_words_keeps_ascii_contractions_whole() -> None: + """ASCII contractions become one token instead of a word plus a stray letter. + + This is the one place where relaxed terms differ from the previous ASCII + behaviour. It only ever lowers the token count, so no query that the guards + used to reject can start relaxing because of it. + """ + assert relaxed_query_words("don't touch this") == ["don't", "touch"] diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py new file mode 100644 index 000000000..a1339320a --- /dev/null +++ b/tests/repository/test_search_relaxed_rendering.py @@ -0,0 +1,85 @@ +"""Relaxed-fallback rendering must survive the tokens the eligibility helper emits.""" + +import sqlite3 + +import pytest + +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository + +CREATE_FTS = ( + "CREATE VIRTUAL TABLE t USING fts5(" + "body, tokenize='unicode61 tokenchars 0x2F', prefix='1,2,3,4')" +) +DOCUMENT = ( + "don't touch this п’ять проектів об'єкт доступу как отозвать выданный доступ पहुंच कैसे रद्द करें" +) + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", '"don\'t"* OR touch*'), + ("скасувати об'єкт виданий доступ", 'скасувати* OR "об\'єкт"* OR виданий* OR доступ*'), + ("п’ять виданих різних об’єктів", "п’ять* OR виданих* OR різних* OR об’єктів*"), + ("how to revoke granted access", "revoke* OR granted* OR access*"), + ], +) +def test_sqlite_relaxed_text_quotes_only_terms_that_need_it(query: str, expected: str) -> None: + """Apostrophe terms are quoted; every other term renders exactly as before.""" + assert SQLiteSearchRepository._relaxed_fts_text(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "don't touch this", + "скасувати об'єкт виданий доступ", + "п’ять виданих різних об’єктів", + "как отозвать выданный доступ", + "पहुंच कैसे रद्द करें", + ], +) +def test_sqlite_relaxed_text_is_accepted_by_fts5(query: str) -> None: + """The rendered expression must parse. + + An unquoted apostrophe raises `fts5: syntax error`, which the repository + catches and turns into an empty result — the relaxed retry then silently + contributes nothing, which is the failure this fallback exists to prevent. + """ + relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + assert relaxed is not None + + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression matched nothing: {relaxed}" + + +def test_sqlite_relaxed_text_bare_apostrophe_would_be_rejected() -> None: + """Pin why the quoting exists, so removing it fails loudly rather than silently.""" + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + with pytest.raises(sqlite3.OperationalError, match="fts5: syntax error"): + connection.execute("SELECT rowid FROM t WHERE t MATCH ?", ("don't* OR touch*",)) + finally: + connection.close() + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", "'don''t':* | touch:*"), + ("скасувати об'єкт виданий доступ", "скасувати:* | 'об''єкт':* | виданий:* | доступ:*"), + ("how to revoke granted access", "revoke:* | granted:* | access:*"), + ], +) +def test_postgres_relaxed_tsquery_quotes_apostrophe_lexemes(query: str, expected: str) -> None: + """Postgres carries the same token shapes, so it needs the same escaping.""" + assert PostgresSearchRepository._relaxed_tsquery_text(query) == expected