fix(core): count non-Latin tokens when relaxing full-text queries

src/basic_memory/repository/postgres_search_repository.py

@@ -230,13 +230,25 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
         # For non-Boolean queries, prepare single term
         return self._prepare_single_term(term, is_prefix)
 
+    @staticmethod
+    def _relaxed_tsquery_term(word: str) -> str:
+        """Render one relaxed word as a tsquery-safe prefix expression.
+
+        Mirrors the SQLite renderer: a word token can contain an apostrophe, and
+        tsquery reads that as lexeme-quoting syntax rather than text. Quoting the
+        lexeme and doubling any interior quote keeps it literal.
+        """
+        if "'" in word:
+            return "'{}':*".format(word.replace("'", "''"))
+        return f"{word}:*"
+
     @staticmethod
     def _relaxed_tsquery_text(search_text: Optional[str]) -> Optional[str]:
         """OR-relaxed tsquery expression for a failed strict query, or None."""
         words = relaxed_query_words(search_text)
         if not words:
             return None
-        return " | ".join(f"{word}:*" for word in words)
+        return " | ".join(PostgresSearchRepository._relaxed_tsquery_term(word) for word in words)
 
     def _prepare_boolean_query(self, query: str) -> str:
         """Convert Boolean query to tsquery format.

src/basic_memory/repository/search_query.py

@@ -1,6 +1,7 @@
 """Shared full-text query preparation rules."""
 
 import re
+import unicodedata
 
 # Interrogative/function words contribute lexical noise when a strict
 # full-text query is relaxed: "when OR did OR a" matches loud wrong documents
@@ -25,8 +26,71 @@
     r"\uff65-\uff9f"  # Halfwidth Katakana
     r"]"
 )
-RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+")
 RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、"
+# Written inside a word (Persian "\u200c", Indic conjuncts) rather than between words.
+RELAXATION_JOIN_CONTROLS = "\u200c\u200d"
+# Word-internal only between letters: "\u043f\u2019\u044f\u0442\u044c", "don't" \u2014 but not "SPEC 16's",
+# where the digit must stay its own token so the numeric guard still sees it.
+RELAXATION_WORD_INTERNAL_PUNCTUATION = "'\u2019"
+
+
+def _is_token_continuation(text: str, index: int, current: list[str]) -> bool:
+    """Whether a non-alphanumeric character belongs to the word being read.
+
+    Combining marks, zero-width join controls, and apostrophes are written inside
+    a word but are not alphanumeric, so a naive scan treats them as separators
+    and splits one orthographic word into several tokens.
+
+    An apostrophe counts only between two letters. That keeps "\u043f\u2019\u044f\u0442\u044c" whole while
+    leaving "SPEC 16's" split, so the digit stays a token of its own and the
+    numeric-identifier guard still rejects the query.
+    """
+    char = text[index]
+    if char in RELAXATION_JOIN_CONTROLS or unicodedata.category(char).startswith("M"):
+        return True
+    if char in RELAXATION_WORD_INTERNAL_PUNCTUATION:
+        follows_letter = bool(current) and current[-1].isalpha()
+        precedes_letter = index + 1 < len(text) and text[index + 1].isalpha()
+        return follows_letter and precedes_letter
+    return False
+
+
+def relaxation_word_tokens(text: str) -> list[str]:
+    """Split text into word tokens for the relaxation eligibility guards.
+
+    A token is a run of alphanumeric characters together with the combining
+    marks, join controls, and apostrophes written inside it. Counting this way matters because
+    an ASCII-only rule saw zero tokens in Cyrillic, Greek, Hebrew, Arabic,
+    Armenian, and Georgian queries, so the three-token guard below rejected every
+    one of them and the hybrid FTS branch silently contributed nothing.
+
+    Counting characters that live inside a word as separators is just as wrong in
+    the other direction: it cuts abugidas (Devanagari, Thai), decomposed text,
+    and Persian or Indic words joined by U+200C/U+200D into fragments. One word
+    then looks like several tokens, clears the three-token guard, and relaxes
+    into a broad OR of fragments — the opposite of what the guard is for.
+    """
+    tokens: list[str] = []
+    current: list[str] = []
+
+    def flush() -> None:
+        # Trailing join controls are word-internal by definition, so a token that
+        # ends in one is really a word followed by a separator.
+        token = "".join(current).rstrip(RELAXATION_JOIN_CONTROLS)
+        if token:
+            tokens.append(token)
+        current.clear()
+
+    for index, char in enumerate(text):
+        # A leading mark, join control, or apostrophe has no base character to
+        # attach to, so it cannot open a token; that keeps stray punctuation from
+        # forming fragment-only terms.
+        if char.isalnum() or (current and _is_token_continuation(text, index, current)):
+            current.append(char)
+        elif current:
+            flush()
+    flush()
+    return tokens
 
 
 def _dedupe_relaxation_words(words: list[str]) -> list[str]:
@@ -57,13 +121,16 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None:
 
     - empty / quoted / explicit-boolean queries (user intent is not
       second-guessed);
-    - fewer than three alphanumeric tokens (short queries like "New Feature"
+    - fewer than three word tokens (short queries like "New Feature"
       over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize
-      to 1.0 and can outrank the vector result the user wanted);
+      to 1.0 and can outrank the vector result the user wanted). Tokens are
+      counted with relaxation_word_tokens, so scripts other than Latin reach the
+      same guard instead of being read as zero tokens;
     - CJK terms separated by whitespace can relax with two or more terms because
-      the ASCII token gate would otherwise suppress the fallback entirely;
-    - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries
-      over-broaden and create false positives under OR.
+      they are not whitespace-delimited the way the token guard assumes;
+    - any numeric token ("root note 1", "SPEC 16", "SPEC \u216b") — identifier-like
+      queries over-broaden and create false positives under OR. Numeric-ness is
+      Unicode-wide, so Nl/No characters such as \u216b and \u00bd are caught too.
     """
     if not search_text:
         return None
@@ -77,7 +144,7 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None:
     has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words)
 
     if has_cjk_term:
-        if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words):
+        if len(cjk_words) < 2 or any(word.isnumeric() for word in cjk_words):
             return None
         pruned_words = [
             word
@@ -91,8 +158,8 @@ def relaxed_query_words(search_text: str | None) -> list[str] | None:
         # Outcome: preserve the short-query guard after pruning to avoid a broad retry.
         return relaxed_words if len(relaxed_words) >= 2 else None
 
-    tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower())
-    if len(tokens) < 3 or any(token.isdigit() for token in tokens):
+    tokens = relaxation_word_tokens(stripped.lower())
+    if len(tokens) < 3 or any(token.isnumeric() for token in tokens):
         return None
     pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS]
     return _dedupe_relaxation_words(pruned_words or tokens) or None

src/basic_memory/repository/sqlite_search_repository.py

@@ -397,13 +397,26 @@ def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
         # For non-Boolean queries, use the single term preparation logic
         return self._prepare_single_term(term, is_prefix)
 
+    @staticmethod
+    def _relaxed_fts_term(word: str) -> str:
+        """Render one relaxed word as an FTS5-safe prefix expression.
+
+        A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated
+        bare it is FTS5 syntax, not text: the whole expression fails to parse, the
+        caller swallows the syntax error, and the relaxed retry returns nothing —
+        the exact silent-empty-FTS failure this fallback exists to prevent.
+        """
+        if "'" in word or '"' in word:
+            return '"{}"*'.format(word.replace('"', '""'))
+        return f"{word}*"
+
     @staticmethod
     def _relaxed_fts_text(search_text: Optional[str]) -> Optional[str]:
         """OR-relaxed FTS5 expression for a failed strict query, or None."""
         words = relaxed_query_words(search_text)
         if not words:
             return None
-        return " OR ".join(f"{word}*" for word in words)
+        return " OR ".join(SQLiteSearchRepository._relaxed_fts_term(word) for word in words)
 
     @override
     async def semantic_effectively_enabled(self) -> bool:

tests/repository/test_search_relaxation.py

@@ -34,3 +34,154 @@ def test_relaxed_query_words_supports_whitespace_separated_cjk_scripts(
 def test_relaxed_query_words_preserves_short_query_guard_after_cjk_pruning(query: str) -> None:
     """Unsafe, duplicate, or stopword terms cannot pad a one-term CJK relaxation."""
     assert relaxed_query_words(query) is None
+
+
+@pytest.mark.parametrize(
+    ("query", "expected"),
+    [
+        ("как отозвать выданный доступ", ["как", "отозвать", "выданный", "доступ"]),
+        ("як відкликати виданий доступ", ["як", "відкликати", "виданий", "доступ"]),
+        ("πώς να ανακαλέσετε πρόσβαση", ["πώς", "να", "ανακαλέσετε", "πρόσβαση"]),
+        ("כיצד לבטל גישה שניתנה", ["כיצד", "לבטל", "גישה", "שניתנה"]),
+        ("كيف تلغي الوصول الممنوح", ["كيف", "تلغي", "الوصول", "الممنوح"]),
+        ("ինչպես չեղարկել տրված մուտքը", ["ինչպես", "չեղարկել", "տրված", "մուտքը"]),
+    ],
+)
+def test_relaxed_query_words_supports_non_latin_alphabetic_scripts(
+    query: str,
+    expected: list[str],
+) -> None:
+    """Non-Latin alphabetic queries reach the same guard as Latin ones.
+
+    An ASCII-only token pattern found zero tokens in these queries, so the
+    three-token guard rejected every one of them and the hybrid FTS branch
+    contributed nothing — hybrid search silently became vector-only.
+    """
+    assert relaxed_query_words(query) == expected
+
+
+@pytest.mark.parametrize(
+    ("query", "expected"),
+    [
+        ("पहुंच कैसे रद्द करें", ["पहुंच", "कैसे", "रद्द", "करें"]),  # Devanagari
+        ("วิธี เพิกถอน การเข้าถึง", ["วิธี", "เพิกถอน", "การเข้าถึง"]),  # Thai
+        ("como revogar acesso concedido", ["como", "revogar", "acesso", "concedido"]),
+    ],
+)
+def test_relaxed_query_words_keeps_combining_marks_with_their_base_character(
+    query: str,
+    expected: list[str],
+) -> None:
+    """Vowel signs and diacritics stay inside the word they attach to.
+
+    Combining marks are not alphanumeric, so treating them as separators splits
+    one abugida word into syllable fragments. The token count then inflates past
+    the three-token guard and relaxation ORs those fragments together.
+    """
+    assert relaxed_query_words(query) == expected
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "अंतर्राष्ट्रीयकरण",  # one Devanagari word: 7 fragments if marks split it
+        "การเข้าถึง",  # one Thai word
+        "pre\u0301sentation",  # one word, NFD-decomposed acute accent
+    ],
+)
+def test_relaxed_query_words_guards_single_words_with_combining_marks(query: str) -> None:
+    """A single word stays one token, so the short-query guard still rejects it."""
+    assert relaxed_query_words(query) is None
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "отозвать доступ",  # fewer than three tokens
+        "спека 16 доступ",  # pure-digit token
+        '"точная фраза"',  # quoted: user intent is explicit
+        "доступ OR токен",  # explicit boolean: user intent is explicit
+    ],
+)
+def test_relaxed_query_words_applies_existing_guards_to_non_latin(query: str) -> None:
+    """Non-Latin queries gain no exemption from the short-query and identifier guards."""
+    assert relaxed_query_words(query) is None
+
+
+@pytest.mark.parametrize(
+    ("query", "expected"),
+    [
+        ("می‌روم خانه", None),  # two Persian words, one joined by ZWNJ
+        ("نمی‌خواهم دسترسی را لغو", ["نمی‌خواهم", "دسترسی", "را", "لغو"]),
+        ("क‍ष विशेष पहुंच", ["क‍ष", "विशेष", "पहुंच"]),  # explicit ZWJ conjunct
+    ],
+)
+def test_relaxed_query_words_keeps_join_controls_inside_words(
+    query: str,
+    expected: list[str] | None,
+) -> None:
+    """U+200C/U+200D are written inside a word, so they must not split its token.
+
+    Splitting on them inflates the token count: a two-word Persian query looks
+    like three tokens, clears the three-token guard, and relaxes into fragments.
+    """
+    assert relaxed_query_words(query) == expected
+
+
+@pytest.mark.parametrize(
+    "query",
+    [
+        "SPEC Ⅻ design",  # Nl: Roman numeral twelve
+        "spec ½ design",  # No: vulgar fraction one half
+        "٣ ٤ ٥",  # Arabic-Indic digits
+    ],
+)
+def test_relaxed_query_words_rejects_unicode_numeric_tokens(query: str) -> None:
+    """The identifier guard classifies numbers Unicode-wide, not just as ASCII digits.
+
+    `isdigit()` is false for Nl/No characters, so admitting every alphanumeric
+    character would let identifier-like queries slip past the numeric guard.
+    """
+    assert relaxed_query_words(query) is None
+
+
+@pytest.mark.parametrize(
+    ("query", "expected"),
+    [
+        ("п’ять проектів", None),  # two Ukrainian words, U+2019
+        ("об'єкт доступу", None),  # two Ukrainian words, ASCII apostrophe
+        (
+            "скасувати п’ять виданих об'єктів",
+            ["скасувати", "п’ять", "виданих", "об'єктів"],
+        ),
+    ],
+)
+def test_relaxed_query_words_keeps_apostrophes_inside_words(
+    query: str,
+    expected: list[str] | None,
+) -> None:
+    """A word-internal apostrophe must not split one word into several tokens.
+
+    Splitting on it turned a two-word Ukrainian query into three tokens, which
+    cleared the three-token guard and relaxed into one-letter fragments.
+    """
+    assert relaxed_query_words(query) == expected
+
+
+def test_relaxed_query_words_apostrophe_does_not_shield_numeric_tokens() -> None:
+    """An apostrophe joins letters only, so a digit stays a token of its own.
+
+    Were `16's` read as one token it would not be numeric, and the query would
+    escape the identifier guard that rejects `SPEC 16 design`.
+    """
+    assert relaxed_query_words("SPEC 16's design") is None
+
+
+def test_relaxed_query_words_keeps_ascii_contractions_whole() -> None:
+    """ASCII contractions become one token instead of a word plus a stray letter.
+
+    This is the one place where relaxed terms differ from the previous ASCII
+    behaviour. It only ever lowers the token count, so no query that the guards
+    used to reject can start relaxing because of it.
+    """
+    assert relaxed_query_words("don't touch this") == ["don't", "touch"]