From 57c1a882d39c17cb075d24886e1b69a7a346a2a8 Mon Sep 17 00:00:00 2001 From: gingeard Date: Mon, 17 Aug 2026 16:27:59 +0100 Subject: [PATCH 1/5] fix(core): count non-Latin tokens when relaxing full-text queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `relaxed_query_words` decides whether a strict AND full-text query may be retried as an OR query. Its eligibility check counted tokens with `[A-Za-z0-9]+`, so any query written in a non-Latin alphabet produced zero tokens, tripped the "fewer than three tokens" guard, and never relaxed. Because only the hybrid path opts into relaxation, the effect was a silent degradation rather than an error: the FTS branch returned nothing for question-form queries, score fusion had a single non-zero side, and hybrid search became vector-only ranking. Nothing logs at default level, so the lexical half of hybrid search is simply absent for these languages. A dedicated CJK branch already worked around the same gate for Han, kana, and Hangul, which suggests the ASCII assumption was known but only patched for one script family. Switch the pattern to a Unicode-aware `[^\W_]+`. This keeps the alphanumeric intent (underscore stays excluded) and leaves every existing guard in place: short queries, quoted queries, explicit booleans, and pure-digit identifiers are rejected exactly as before, for Latin and non-Latin alike. Verified against a Russian corpus of 17 notes and 176 observations: before the change `fts_count=0` on every question-form query; after it the FTS branch contributes candidates and fusion has two sides again. Abugidas remain partially handled — Devanagari and Thai vowel signs are non-spacing marks outside `\w`, so words split into syllable fragments. Relaxation engages, but the OR terms are fragments; a test pins that behaviour so a future fix is deliberate. Tests: 12 added (6 alphabetic scripts relax, 4 guards still reject, 2 pin the abugida limitation). Existing search suites unchanged — 677 → 689 passing, same 32 pre-existing environment-dependent failures. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 19 +++++-- tests/repository/test_search_relaxation.py | 59 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 8544bdb47..2b56b911f 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -25,7 +25,16 @@ r"\uff65-\uff9f" # Halfwidth Katakana r"]" ) -RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+") +# Unicode-aware: `\w` minus underscore keeps the alphanumeric intent of the +# original ASCII pattern while also counting Cyrillic, Greek, Hebrew, Arabic, +# Armenian, and Georgian words. An ASCII-only gate made every such query look +# like it had zero tokens, so the three-token guard below rejected all of them +# and the hybrid FTS branch silently contributed nothing. +# +# Abugidas (Devanagari, Thai) are only partially handled: their vowel signs are +# non-spacing marks outside `\w`, so a word splits into syllable fragments. +# Relaxation still engages, but proper support needs grapheme segmentation. +RELAXATION_WORD_TOKEN_PATTERN = re.compile(r"[^\W_]+", re.UNICODE) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" @@ -59,9 +68,11 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: second-guessed); - fewer than three alphanumeric tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize - to 1.0 and can outrank the vector result the user wanted); + to 1.0 and can outrank the vector result the user wanted). Tokens are + counted with a Unicode-aware pattern, so scripts other than Latin reach + the same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because - the ASCII token gate would otherwise suppress the fallback entirely; + they are not whitespace-delimited the way the token guard assumes; - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries over-broaden and create false positives under OR. """ @@ -91,7 +102,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: # Outcome: preserve the short-query guard after pruning to avoid a broad retry. return relaxed_words if len(relaxed_words) >= 2 else None - tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower()) + tokens = RELAXATION_WORD_TOKEN_PATTERN.findall(stripped.lower()) if len(tokens) < 3 or any(token.isdigit() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9a7181d8b..01f0c2f9d 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -34,3 +34,62 @@ def test_relaxed_query_words_supports_whitespace_separated_cjk_scripts( def test_relaxed_query_words_preserves_short_query_guard_after_cjk_pruning(query: str) -> None: """Unsafe, duplicate, or stopword terms cannot pad a one-term CJK relaxation.""" assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("как отозвать выданный доступ", ["как", "отозвать", "выданный", "доступ"]), + ("як відкликати виданий доступ", ["як", "відкликати", "виданий", "доступ"]), + ("πώς να ανακαλέσετε πρόσβαση", ["πώς", "να", "ανακαλέσετε", "πρόσβαση"]), + ("כיצד לבטל גישה שניתנה", ["כיצד", "לבטל", "גישה", "שניתנה"]), + ("كيف تلغي الوصول الممنوح", ["كيف", "تلغي", "الوصول", "الممنوح"]), + ("ինչպես չեղարկել տրված մուտքը", ["ինչպես", "չեղարկել", "տրված", "մուտքը"]), + ], +) +def test_relaxed_query_words_supports_non_latin_alphabetic_scripts( + query: str, + expected: list[str], +) -> None: + """Non-Latin alphabetic queries reach the same guard as Latin ones. + + An ASCII-only token pattern found zero tokens in these queries, so the + three-token guard rejected every one of them and the hybrid FTS branch + contributed nothing — hybrid search silently became vector-only. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "पहुंच कैसे रद्द करें", # Devanagari + "วิธี เพิกถอน การเข้าถึง", # Thai + ], +) +def test_relaxed_query_words_still_splits_scripts_with_combining_marks(query: str) -> None: + """Known limitation: abugidas split on combining marks, which `\\w` excludes. + + Vowel signs and viramas are non-spacing marks, so a `\\w`-based pattern cuts + each syllable cluster into fragments. Relaxation still engages — the token + count only grows — but the resulting OR terms are word fragments rather than + words. Proper support needs grapheme-cluster segmentation; this test pins the + current behaviour so a future change is a deliberate one. + """ + words = relaxed_query_words(query) + assert words is not None + assert len(words) > len(query.split()) + + +@pytest.mark.parametrize( + "query", + [ + "отозвать доступ", # fewer than three tokens + "спека 16 доступ", # pure-digit token + '"точная фраза"', # quoted: user intent is explicit + "доступ OR токен", # explicit boolean: user intent is explicit + ], +) +def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None: + """Non-Latin queries gain no exemption from the short-query and identifier guards.""" + assert relaxed_query_words(query) is None From 5576b5fb9c3b51c186c166f4d65f4acfa7dba0a0 Mon Sep 17 00:00:00 2001 From: gingeard Date: Mon, 17 Aug 2026 16:49:51 +0100 Subject: [PATCH 2/5] fix(core): keep combining marks with their base character in query tokens Combining marks are not alphanumeric, so counting them as token separators split abugida words (Devanagari, Thai) and NFD-decomposed text into syllable fragments. A single word then looked like several tokens, passed the three-token guard, and relaxed into a broad OR of one- and two-letter fragments whose top FTS row normalizes to 1.0 during hybrid fusion. Group marks with the base character they attach to. Single words in those scripts now stay one token and the short-query guard rejects them as intended, while multi-word queries relax into whole words. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 49 +++++++++++++++------ tests/repository/test_search_relaxation.py | 41 +++++++++++------ 2 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 2b56b911f..6954d1dd3 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -1,6 +1,7 @@ """Shared full-text query preparation rules.""" import re +import unicodedata # Interrogative/function words contribute lexical noise when a strict # full-text query is relaxed: "when OR did OR a" matches loud wrong documents @@ -25,19 +26,39 @@ r"\uff65-\uff9f" # Halfwidth Katakana r"]" ) -# Unicode-aware: `\w` minus underscore keeps the alphanumeric intent of the -# original ASCII pattern while also counting Cyrillic, Greek, Hebrew, Arabic, -# Armenian, and Georgian words. An ASCII-only gate made every such query look -# like it had zero tokens, so the three-token guard below rejected all of them -# and the hybrid FTS branch silently contributed nothing. -# -# Abugidas (Devanagari, Thai) are only partially handled: their vowel signs are -# non-spacing marks outside `\w`, so a word splits into syllable fragments. -# Relaxation still engages, but proper support needs grapheme segmentation. -RELAXATION_WORD_TOKEN_PATTERN = re.compile(r"[^\W_]+", re.UNICODE) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" +def relaxation_word_tokens(text: str) -> list[str]: + """Split text into word tokens for the relaxation eligibility guards. + + A token is a run of alphanumeric characters together with any combining + marks attached to them. Counting this way matters twice over: + + - an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, + Armenian, and Georgian queries, so the three-token guard below rejected + every one of them and the hybrid FTS branch silently contributed nothing; + - combining marks are not alphanumeric, so counting them as separators cuts + abugidas (Devanagari, Thai) and decomposed text into syllable fragments. + One word then looks like several tokens, passes the three-token guard, and + relaxes into a broad OR of fragments — the opposite of what the guard is + for. Keeping marks with their base character preserves it. + """ + tokens: list[str] = [] + current: list[str] = [] + for char in text: + # A leading mark has no base character to attach to, so it cannot open + # a token; that keeps stray marks from forming fragment-only terms. + if char.isalnum() or (current and unicodedata.category(char).startswith("M")): + current.append(char) + elif current: + tokens.append("".join(current)) + current = [] + if current: + tokens.append("".join(current)) + return tokens + + def _dedupe_relaxation_words(words: list[str]) -> list[str]: """Preserve first-seen relaxed terms while removing duplicates case-insensitively.""" deduped_terms: list[str] = [] @@ -66,11 +87,11 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: - empty / quoted / explicit-boolean queries (user intent is not second-guessed); - - fewer than three alphanumeric tokens (short queries like "New Feature" + - fewer than three word tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize to 1.0 and can outrank the vector result the user wanted). Tokens are - counted with a Unicode-aware pattern, so scripts other than Latin reach - the same guard instead of being read as zero tokens; + counted with relaxation_word_tokens, so scripts other than Latin reach the + same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because they are not whitespace-delimited the way the token guard assumes; - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries @@ -102,7 +123,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: # Outcome: preserve the short-query guard after pruning to avoid a broad retry. return relaxed_words if len(relaxed_words) >= 2 else None - tokens = RELAXATION_WORD_TOKEN_PATTERN.findall(stripped.lower()) + tokens = relaxation_word_tokens(stripped.lower()) if len(tokens) < 3 or any(token.isdigit() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 01f0c2f9d..9554939c5 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -61,24 +61,37 @@ def test_relaxed_query_words_supports_non_latin_alphabetic_scripts( @pytest.mark.parametrize( - "query", + ("query", "expected"), [ - "पहुंच कैसे रद्द करें", # Devanagari - "วิธี เพิกถอน การเข้าถึง", # Thai + ("पहुंच कैसे रद्द करें", ["पहुंच", "कैसे", "रद्द", "करें"]), # Devanagari + ("วิธี เพิกถอน การเข้าถึง", ["วิธี", "เพิกถอน", "การเข้าถึง"]), # Thai + ("como revogar acesso concedido", ["como", "revogar", "acesso", "concedido"]), ], ) -def test_relaxed_query_words_still_splits_scripts_with_combining_marks(query: str) -> None: - """Known limitation: abugidas split on combining marks, which `\\w` excludes. - - Vowel signs and viramas are non-spacing marks, so a `\\w`-based pattern cuts - each syllable cluster into fragments. Relaxation still engages — the token - count only grows — but the resulting OR terms are word fragments rather than - words. Proper support needs grapheme-cluster segmentation; this test pins the - current behaviour so a future change is a deliberate one. +def test_relaxed_query_words_keeps_combining_marks_with_their_base_character( + query: str, + expected: list[str], +) -> None: + """Vowel signs and diacritics stay inside the word they attach to. + + Combining marks are not alphanumeric, so treating them as separators splits + one abugida word into syllable fragments. The token count then inflates past + the three-token guard and relaxation ORs those fragments together. """ - words = relaxed_query_words(query) - assert words is not None - assert len(words) > len(query.split()) + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "अंतर्राष्ट्रीयकरण", # one Devanagari word: 7 fragments if marks split it + "การเข้าถึง", # one Thai word + "pre\u0301sentation", # one word, NFD-decomposed acute accent + ], +) +def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str) -> None: + """A single word stays one token, so the short-query guard still rejects it.""" + assert relaxed_query_words(query) is None @pytest.mark.parametrize( From 88b2453912f215879e08c60d0fc182d19ec8f387 Mon Sep 17 00:00:00 2001 From: gingeard Date: Tue, 18 Aug 2026 11:35:46 +0100 Subject: [PATCH 3/5] fix(core): keep join controls in tokens and reject Unicode numeric tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two eligibility-guard regressions from widening token recognition beyond ASCII. U+200C and U+200D are written inside an orthographic word, so terminating a token on them split Persian words and explicit Devanagari ZWJ conjuncts: the two-word query "می‌روم خانه" became three tokens and cleared the three-token guard. Join controls are now read as word-internal, and a trailing one is treated as a separator rather than kept in the term. isdigit() is false for Nl/No characters, so admitting every alphanumeric character let "SPEC Ⅻ design" and "spec ½ design" past the numeric-identifier guard that the old ASCII path rejected as too short. Both guards now classify numbers Unicode-wide with isnumeric(); the CJK branch is aligned for the same reason, since it carried the identical hole. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 65 ++++++++++++++------- tests/repository/test_search_relaxation.py | 37 ++++++++++++ 2 files changed, 80 insertions(+), 22 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 6954d1dd3..9d826a152 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -27,35 +27,55 @@ r"]" ) RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" +# Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. +RELAXATION_JOIN_CONTROLS = "\u200c\u200d" + + +def _is_token_continuation(char: str) -> bool: + """Whether a non-alphanumeric character belongs to the word being read. + + Combining marks and the zero-width join controls are written inside a word + but are not alphanumeric, so a naive scan would treat them as separators and + split one orthographic word into several tokens. + """ + return char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M") def relaxation_word_tokens(text: str) -> list[str]: """Split text into word tokens for the relaxation eligibility guards. - A token is a run of alphanumeric characters together with any combining - marks attached to them. Counting this way matters twice over: - - - an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, - Armenian, and Georgian queries, so the three-token guard below rejected - every one of them and the hybrid FTS branch silently contributed nothing; - - combining marks are not alphanumeric, so counting them as separators cuts - abugidas (Devanagari, Thai) and decomposed text into syllable fragments. - One word then looks like several tokens, passes the three-token guard, and - relaxes into a broad OR of fragments — the opposite of what the guard is - for. Keeping marks with their base character preserves it. + A token is a run of alphanumeric characters together with the combining + marks and join controls written inside it. Counting this way matters because + an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, + Armenian, and Georgian queries, so the three-token guard below rejected every + one of them and the hybrid FTS branch silently contributed nothing. + + Counting characters that live inside a word as separators is just as wrong in + the other direction: it cuts abugidas (Devanagari, Thai), decomposed text, + and Persian or Indic words joined by U+200C/U+200D into fragments. One word + then looks like several tokens, clears the three-token guard, and relaxes + into a broad OR of fragments — the opposite of what the guard is for. """ tokens: list[str] = [] current: list[str] = [] + + def flush() -> None: + # Trailing join controls are word-internal by definition, so a token that + # ends in one is really a word followed by a separator. + token = "".join(current).rstrip(RELAXATION_JOIN_CONTROLS) + if token: + tokens.append(token) + current.clear() + for char in text: - # A leading mark has no base character to attach to, so it cannot open - # a token; that keeps stray marks from forming fragment-only terms. - if char.isalnum() or (current and unicodedata.category(char).startswith("M")): + # A leading mark or join control has no base character to attach to, so + # it cannot open a token; that keeps stray marks from forming + # fragment-only terms. + if char.isalnum() or (current and _is_token_continuation(char)): current.append(char) elif current: - tokens.append("".join(current)) - current = [] - if current: - tokens.append("".join(current)) + flush() + flush() return tokens @@ -94,8 +114,9 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: same guard instead of being read as zero tokens; - CJK terms separated by whitespace can relax with two or more terms because they are not whitespace-delimited the way the token guard assumes; - - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries - over-broaden and create false positives under OR. + - any numeric token ("root note 1", "SPEC 16", "SPEC \u216b") — identifier-like + queries over-broaden and create false positives under OR. Numeric-ness is + Unicode-wide, so Nl/No characters such as \u216b and \u00bd are caught too. """ if not search_text: return None @@ -109,7 +130,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) if has_cjk_term: - if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): + if len(cjk_words) < 2 or any(word.isnumeric() for word in cjk_words): return None pruned_words = [ word @@ -124,7 +145,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None: return relaxed_words if len(relaxed_words) >= 2 else None tokens = relaxation_word_tokens(stripped.lower()) - if len(tokens) < 3 or any(token.isdigit() for token in tokens): + if len(tokens) < 3 or any(token.isnumeric() for token in tokens): return None pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] return _dedupe_relaxation_words(pruned_words or tokens) or None diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index 9554939c5..eeb299483 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -106,3 +106,40 @@ def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None: """Non-Latin queries gain no exemption from the short-query and identifier guards.""" assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("می‌روم خانه", None), # two Persian words, one joined by ZWNJ + ("نمی‌خواهم دسترسی را لغو", ["نمی‌خواهم", "دسترسی", "را", "لغو"]), + ("क‍ष विशेष पहुंच", ["क‍ष", "विशेष", "पहुंच"]), # explicit ZWJ conjunct + ], +) +def test_relaxed_query_words_keeps_join_controls_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """U+200C/U+200D are written inside a word, so they must not split its token. + + Splitting on them inflates the token count: a two-word Persian query looks + like three tokens, clears the three-token guard, and relaxes into fragments. + """ + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "SPEC Ⅻ design", # Nl: Roman numeral twelve + "spec ½ design", # No: vulgar fraction one half + "٣ ٤ ٥", # Arabic-Indic digits + ], +) +def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None: + """The identifier guard classifies numbers Unicode-wide, not just as ASCII digits. + + `isdigit()` is false for Nl/No characters, so admitting every alphanumeric + character would let identifier-like queries slip past the numeric guard. + """ + assert relaxed_query_words(query) is None From 401d5d45cc767e78b1c0967bd6b900dcafa030fd Mon Sep 17 00:00:00 2001 From: gingeard Date: Tue, 18 Aug 2026 12:04:27 +0100 Subject: [PATCH 4/5] fix(core): keep word-internal apostrophes inside query tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splitting on an apostrophe cut Ukrainian words apart: the two-word query "п’ять проектів" became three tokens, cleared the three-token guard, and relaxed into an OR containing one-letter fragments. Both U+2019 and the ASCII apostrophe are affected. An apostrophe now continues a token only between two letters. That keeps "SPEC 16's design" split on the digit, so the numeric-identifier guard still rejects it, and it leaves a leading or trailing apostrophe as a separator. One consequence for ASCII input: contractions become a single token, so "don't touch this" yields ["don't", "touch"] where it previously yielded ["don", "t", "touch"]. Merging can only lower the token count, so no query the guards used to reject can begin relaxing because of it, and the hyphen and slash cases named in #1022 are unchanged. Signed-off-by: gingeard --- src/basic_memory/repository/search_query.py | 36 ++++++++++++------ tests/repository/test_search_relaxation.py | 42 +++++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/basic_memory/repository/search_query.py b/src/basic_memory/repository/search_query.py index 9d826a152..0f5e10148 100644 --- a/src/basic_memory/repository/search_query.py +++ b/src/basic_memory/repository/search_query.py @@ -29,23 +29,37 @@ RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、" # Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words. RELAXATION_JOIN_CONTROLS = "\u200c\u200d" +# Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's", +# where the digit must stay its own token so the numeric guard still sees it. +RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019" -def _is_token_continuation(char: str) -> bool: +def _is_token_continuation(text: str, index: int, current: list[str]) -> bool: """Whether a non-alphanumeric character belongs to the word being read. - Combining marks and the zero-width join controls are written inside a word - but are not alphanumeric, so a naive scan would treat them as separators and - split one orthographic word into several tokens. + Combining marks, zero-width join controls, and apostrophes are written inside + a word but are not alphanumeric, so a naive scan treats them as separators + and splits one orthographic word into several tokens. + + An apostrophe counts only between two letters. That keeps "\u043f\u2019\u044f\u0442\u044c" whole while + leaving "SPEC 16's" split, so the digit stays a token of its own and the + numeric-identifier guard still rejects the query. """ - return char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M") + char = text[index] + if char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M"): + return True + if char in RELAXATION_WORD_INTERNAL_PUNCTUATION: + follows_letter = bool(current) and current[-1].isalpha() + precedes_letter = index + 1 < len(text) and text[index + 1].isalpha() + return follows_letter and precedes_letter + return False def relaxation_word_tokens(text: str) -> list[str]: """Split text into word tokens for the relaxation eligibility guards. A token is a run of alphanumeric characters together with the combining - marks and join controls written inside it. Counting this way matters because + marks, join controls, and apostrophes written inside it. Counting this way matters because an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic, Armenian, and Georgian queries, so the three-token guard below rejected every one of them and the hybrid FTS branch silently contributed nothing. @@ -67,11 +81,11 @@ def flush() -> None: tokens.append(token) current.clear() - for char in text: - # A leading mark or join control has no base character to attach to, so - # it cannot open a token; that keeps stray marks from forming - # fragment-only terms. - if char.isalnum() or (current and _is_token_continuation(char)): + for index, char in enumerate(text): + # A leading mark, join control, or apostrophe has no base character to + # attach to, so it cannot open a token; that keeps stray punctuation from + # forming fragment-only terms. + if char.isalnum() or (current and _is_token_continuation(text, index, current)): current.append(char) elif current: flush() diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py index eeb299483..a5d9832a4 100644 --- a/tests/repository/test_search_relaxation.py +++ b/tests/repository/test_search_relaxation.py @@ -143,3 +143,45 @@ def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None: character would let identifier-like queries slip past the numeric guard. """ assert relaxed_query_words(query) is None + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("п’ять проектів", None), # two Ukrainian words, U+2019 + ("об'єкт доступу", None), # two Ukrainian words, ASCII apostrophe + ( + "скасувати п’ять виданих об'єктів", + ["скасувати", "п’ять", "виданих", "об'єктів"], + ), + ], +) +def test_relaxed_query_words_keeps_apostrophes_inside_words( + query: str, + expected: list[str] | None, +) -> None: + """A word-internal apostrophe must not split one word into several tokens. + + Splitting on it turned a two-word Ukrainian query into three tokens, which + cleared the three-token guard and relaxed into one-letter fragments. + """ + assert relaxed_query_words(query) == expected + + +def test_relaxed_query_words_apostrophe_does_not_shield_numeric_tokens() -> None: + """An apostrophe joins letters only, so a digit stays a token of its own. + + Were `16's` read as one token it would not be numeric, and the query would + escape the identifier guard that rejects `SPEC 16 design`. + """ + assert relaxed_query_words("SPEC 16's design") is None + + +def test_relaxed_query_words_keeps_ascii_contractions_whole() -> None: + """ASCII contractions become one token instead of a word plus a stray letter. + + This is the one place where relaxed terms differ from the previous ASCII + behaviour. It only ever lowers the token count, so no query that the guards + used to reject can start relaxing because of it. + """ + assert relaxed_query_words("don't touch this") == ["don't", "touch"] From 1d5ed5d8977a326d59c53c6dce838cab3365eea1 Mon Sep 17 00:00:00 2001 From: gingeard Date: Tue, 18 Aug 2026 12:26:36 +0100 Subject: [PATCH 5/5] fix(core): quote relaxed terms that contain an apostrophe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeping apostrophes inside tokens let them reach the relaxed renderers, which interpolate each word straight into backend query syntax. In FTS5 an ASCII apostrophe is syntax, not text: "don't touch this" rendered as `don't* OR touch*`, which fails to parse with `fts5: syntax error near "'"`. The caller treats a syntax error as an empty result, so the relaxed retry contributed nothing — the same silent-empty-FTS failure this fallback exists to prevent, reintroduced for English contractions and Ukrainian words. Both renderers now quote a word when it carries an apostrophe and leave every other word byte-identical. Postgres gets the matching treatment because it receives the same tokens; its escaping is the documented tsquery form, but I could not exercise it against a live server. Tests cover the rendered expression rather than only the token helper: the SQLite output is executed against a real FTS5 table, and one test pins that the unquoted form raises, so removing the quoting fails loudly. Signed-off-by: gingeard --- .../repository/postgres_search_repository.py | 14 ++- .../repository/sqlite_search_repository.py | 15 +++- .../test_search_relaxed_rendering.py | 85 +++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 tests/repository/test_search_relaxed_rendering.py diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 6d05ca19f..8aa8e5bad 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -230,13 +230,25 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, prepare single term return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_tsquery_term(word: str) -> str: + """Render one relaxed word as a tsquery-safe prefix expression. + + Mirrors the SQLite renderer: a word token can contain an apostrophe, and + tsquery reads that as lexeme-quoting syntax rather than text. Quoting the + lexeme and doubling any interior quote keeps it literal. + """ + if "'" in word: + return "'{}':*".format(word.replace("'", "''")) + return f"{word}:*" + @staticmethod def _relaxed_tsquery_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed tsquery expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " | ".join(f"{word}:*" for word in words) + return " | ".join(PostgresSearchRepository._relaxed_tsquery_term(word) for word in words) def _prepare_boolean_query(self, query: str) -> str: """Convert Boolean query to tsquery format. diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 91bcae435..9dced81f9 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -397,13 +397,26 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: # For non-Boolean queries, use the single term preparation logic return self._prepare_single_term(term, is_prefix) + @staticmethod + def _relaxed_fts_term(word: str) -> str: + """Render one relaxed word as an FTS5-safe prefix expression. + + A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated + bare it is FTS5 syntax, not text: the whole expression fails to parse, the + caller swallows the syntax error, and the relaxed retry returns nothing — + the exact silent-empty-FTS failure this fallback exists to prevent. + """ + if "'" in word or '"' in word: + return '"{}"*'.format(word.replace('"', '""')) + return f"{word}*" + @staticmethod def _relaxed_fts_text(search_text: Optional[str]) -> Optional[str]: """OR-relaxed FTS5 expression for a failed strict query, or None.""" words = relaxed_query_words(search_text) if not words: return None - return " OR ".join(f"{word}*" for word in words) + return " OR ".join(SQLiteSearchRepository._relaxed_fts_term(word) for word in words) @override async def semantic_effectively_enabled(self) -> bool: diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py new file mode 100644 index 000000000..a1339320a --- /dev/null +++ b/tests/repository/test_search_relaxed_rendering.py @@ -0,0 +1,85 @@ +"""Relaxed-fallback rendering must survive the tokens the eligibility helper emits.""" + +import sqlite3 + +import pytest + +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository + +CREATE_FTS = ( + "CREATE VIRTUAL TABLE t USING fts5(" + "body, tokenize='unicode61 tokenchars 0x2F', prefix='1,2,3,4')" +) +DOCUMENT = ( + "don't touch this п’ять проектів об'єкт доступу как отозвать выданный доступ पहुंच कैसे रद्द करें" +) + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", '"don\'t"* OR touch*'), + ("скасувати об'єкт виданий доступ", 'скасувати* OR "об\'єкт"* OR виданий* OR доступ*'), + ("п’ять виданих різних об’єктів", "п’ять* OR виданих* OR різних* OR об’єктів*"), + ("how to revoke granted access", "revoke* OR granted* OR access*"), + ], +) +def test_sqlite_relaxed_text_quotes_only_terms_that_need_it(query: str, expected: str) -> None: + """Apostrophe terms are quoted; every other term renders exactly as before.""" + assert SQLiteSearchRepository._relaxed_fts_text(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "don't touch this", + "скасувати об'єкт виданий доступ", + "п’ять виданих різних об’єктів", + "как отозвать выданный доступ", + "पहुंच कैसे रद्द करें", + ], +) +def test_sqlite_relaxed_text_is_accepted_by_fts5(query: str) -> None: + """The rendered expression must parse. + + An unquoted apostrophe raises `fts5: syntax error`, which the repository + catches and turns into an empty result — the relaxed retry then silently + contributes nothing, which is the failure this fallback exists to prevent. + """ + relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + assert relaxed is not None + + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() + finally: + connection.close() + assert rows, f"relaxed expression matched nothing: {relaxed}" + + +def test_sqlite_relaxed_text_bare_apostrophe_would_be_rejected() -> None: + """Pin why the quoting exists, so removing it fails loudly rather than silently.""" + connection = sqlite3.connect(":memory:") + try: + connection.execute(CREATE_FTS) + connection.execute("INSERT INTO t VALUES (?)", (DOCUMENT,)) + with pytest.raises(sqlite3.OperationalError, match="fts5: syntax error"): + connection.execute("SELECT rowid FROM t WHERE t MATCH ?", ("don't* OR touch*",)) + finally: + connection.close() + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("don't touch this", "'don''t':* | touch:*"), + ("скасувати об'єкт виданий доступ", "скасувати:* | 'об''єкт':* | виданий:* | доступ:*"), + ("how to revoke granted access", "revoke:* | granted:* | access:*"), + ], +) +def test_postgres_relaxed_tsquery_quotes_apostrophe_lexemes(query: str, expected: str) -> None: + """Postgres carries the same token shapes, so it needs the same escaping.""" + assert PostgresSearchRepository._relaxed_tsquery_text(query) == expected