diff --git a/README.md b/README.md index 9081ba8..f672900 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@

## News +* 07/22/2026 [0.0.14](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.14): Auto-fix affected Mistral-family tokenizer regexes, including Laguna S 2.1. * 03/03/2026 [0.0.7](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.7): Fix Qwen 3.5 MoE compat. * 02/09/2026 [0.0.6](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.6): Fix ChatGLM compat. * 09/04/2025 [0.0.5](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.5): Fix `pad_token_id` detection for `LongCat` model. diff --git a/pyproject.toml b/pyproject.toml index f3132d3..65ca576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ build-backend = "setuptools.build_meta" [project] name = "TokeNicer" -version = "0.0.13" +version = "0.0.14" description = "A (nicer) tokenizer you want to use for model `inference` and `training`: with all known peventable `gotchas` normalized or auto-fixed." readme = "README.md" requires-python = ">=3" diff --git a/tests/test_loop_models.py b/tests/test_loop_models.py index cb34b6a..42cc9bc 100644 --- a/tests/test_loop_models.py +++ b/tests/test_loop_models.py @@ -42,6 +42,7 @@ def should_skip_model_error(exc: Exception) -> bool: "No module named", "maximum recursion depth exceeded", "module 'torch' has no attribute 'None'", + "piece must not include null character", ) ) diff --git a/tests/test_mistral_regex.py b/tests/test_mistral_regex.py new file mode 100644 index 0000000..0267b7d --- /dev/null +++ b/tests/test_mistral_regex.py @@ -0,0 +1,90 @@ +# Copyright 2026 ModelCloud.ai +# Copyright 2026 qubitium@modelcloud.ai +# Contact: qubitium@modelcloud.ai, x.com/qubitium +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import Mock, patch + +from huggingface_hub.errors import StrictDataclassClassValidationError + +from tokenicer import Tokenicer + + +class TestMistralRegexNormalization(unittest.TestCase): + @patch("tokenicer.tokenicer.AutoTokenizer.from_pretrained") + def test_mistral_regex_fix_is_enabled_by_default(self, from_pretrained): + expected = object() + from_pretrained.return_value = expected + + tokenizer = Tokenicer._load_tokenizer("poolside/Laguna-S-2.1", trust_remote_code=True) + + self.assertIs(tokenizer, expected) + from_pretrained.assert_called_once_with( + "poolside/Laguna-S-2.1", + trust_remote_code=True, + fix_mistral_regex=True, + ) + + @patch("tokenicer.tokenicer.AutoTokenizer.from_pretrained") + def test_explicit_mistral_regex_setting_is_preserved(self, from_pretrained): + from_pretrained.return_value = object() + + Tokenicer._load_tokenizer("poolside/Laguna-S-2.1", fix_mistral_regex=False) + + from_pretrained.assert_called_once_with( + "poolside/Laguna-S-2.1", + fix_mistral_regex=False, + ) + + @patch("tokenicer.tokenicer.Tokenicer._resolve_tokenizer_class") + @patch("tokenicer.tokenicer.tokenizer_class_name", return_value="FallbackTokenizer") + @patch("tokenicer.tokenicer.tokenizer_special_token_overrides", return_value={"bos_token": ""}) + @patch( + "tokenicer.tokenicer.AutoTokenizer.from_pretrained", + side_effect=StrictDataclassClassValidationError( + validator="validate_layer_type", + cause=ValueError("legacy model config"), + ), + ) + def test_mistral_regex_fix_is_preserved_on_fallback( + self, + auto_from_pretrained, + special_token_overrides, + tokenizer_class_name, + resolve_tokenizer_class, + ): + fallback_from_pretrained = Mock(return_value=object()) + resolve_tokenizer_class.return_value = Mock(from_pretrained=fallback_from_pretrained) + + Tokenicer._load_tokenizer("/tmp/Laguna-S-2.1", trust_remote_code=True) + + auto_from_pretrained.assert_called_once_with( + "/tmp/Laguna-S-2.1", + trust_remote_code=True, + fix_mistral_regex=True, + ) + special_token_overrides.assert_called_once_with("/tmp/Laguna-S-2.1") + tokenizer_class_name.assert_called_once_with("/tmp/Laguna-S-2.1") + resolve_tokenizer_class.assert_called_once_with("/tmp/Laguna-S-2.1", "FallbackTokenizer") + fallback_from_pretrained.assert_called_once_with( + "/tmp/Laguna-S-2.1", + trust_remote_code=True, + fix_mistral_regex=True, + bos_token="", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tokenicer/tokenicer.py b/tokenicer/tokenicer.py index 24e3de1..2896528 100644 --- a/tokenicer/tokenicer.py +++ b/tokenicer/tokenicer.py @@ -28,9 +28,14 @@ from transformers.dynamic_module_utils import get_class_from_dynamic_module try: - from huggingface_hub.errors import StrictDataclassFieldValidationError + from huggingface_hub.errors import StrictDataclassError except Exception: # pragma: no cover - optional dependency path - StrictDataclassFieldValidationError = None + try: + # Compatibility with huggingface_hub versions that expose only the + # concrete field-validation error. + from huggingface_hub.errors import StrictDataclassFieldValidationError as StrictDataclassError + except Exception: + StrictDataclassError = None from .const import DEFAULT_PAD_TOKENS, MODEL_PAD_TOKEN_MAP from .util import ( @@ -54,8 +59,8 @@ KeyError, ) -if StrictDataclassFieldValidationError is not None: - _TOKENIZER_LOAD_EXCEPTIONS = _TOKENIZER_LOAD_EXCEPTIONS + (StrictDataclassFieldValidationError,) +if StrictDataclassError is not None: + _TOKENIZER_LOAD_EXCEPTIONS = _TOKENIZER_LOAD_EXCEPTIONS + (StrictDataclassError,) _KNOWN_LOAD_WARNING_SUPPRESSIONS = [ ( @@ -143,12 +148,18 @@ def load( @staticmethod def _load_tokenizer(pretrained_model_name_or_path: str, **kwargs): Tokenicer._install_tokenizer_compatibility_shims() + load_kwargs = dict(kwargs) + # Let Transformers repair affected Mistral-family pre-tokenizer regexes by + # default. Transformers applies this only when its compatibility detector + # matches, and callers can retain the serialized regex with an explicit + # ``fix_mistral_regex=False``. + load_kwargs.setdefault("fix_mistral_regex", True) try: # Keep the normal Transformers path first so standard checkpoints behave unchanged. - return AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs) + return AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **load_kwargs) except _TOKENIZER_LOAD_EXCEPTIONS: overrides = tokenizer_special_token_overrides(pretrained_model_name_or_path) - retry_kwargs = dict(kwargs) + retry_kwargs = dict(load_kwargs) retry_kwargs.update(overrides) tokenizer_cls_name = tokenizer_class_name(pretrained_model_name_or_path) @@ -185,10 +196,10 @@ def _load_tokenizer(pretrained_model_name_or_path: str, **kwargs): pretrained_model_name_or_path, ) - if kwargs.get("trust_remote_code", False) or not has_custom_tokenizer_code(pretrained_model_name_or_path): + if load_kwargs.get("trust_remote_code", False) or not has_custom_tokenizer_code(pretrained_model_name_or_path): raise - retry_kwargs = dict(kwargs) + retry_kwargs = dict(load_kwargs) retry_kwargs["trust_remote_code"] = True # Local checkpoints with custom tokenizer code can still succeed once remote code is explicitly allowed. logger.warning(