From a6f5cac3f029cc15c9618cf3f0d58420391f2859 Mon Sep 17 00:00:00 2001 From: libaojiang <101562714+libaojiang@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:35:56 +0800 Subject: [PATCH 1/3] fix: support optional tokenizer metadata files --- fastembed/common/preprocessor_utils.py | 33 +++++++++++++----------- tests/test_preprocessor_utils.py | 35 +++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 2fbd9986..40702b08 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -11,7 +11,7 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: tokens_map_path = model_dir / "special_tokens_map.json" if not tokens_map_path.exists(): - raise ValueError(f"Could not find special_tokens_map.json in {model_dir}") + return {} with open(str(tokens_map_path)) as tokens_map_file: tokens_map = json.load(tokens_map_file) @@ -60,8 +60,6 @@ def _resolve_max_context(tokenizer_config: dict[str, Any], model_dir: Path) -> i def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: config_path = model_dir / "config.json" - if not config_path.exists(): - raise ValueError(f"Could not find config.json in {model_dir}") tokenizer_path = model_dir / "tokenizer.json" if not tokenizer_path.exists(): @@ -71,8 +69,11 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: if not tokenizer_config_path.exists(): raise ValueError(f"Could not find tokenizer_config.json in {model_dir}") - with open(str(config_path)) as config_file: - config = json.load(config_file) + has_config = config_path.exists() + config: dict[str, Any] = {} + if has_config: + with open(str(config_path)) as config_file: + config = json.load(config_file) with open(str(tokenizer_config_path)) as tokenizer_config_file: tokenizer_config = json.load(tokenizer_config_file) @@ -93,9 +94,16 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: if pad_token is None: raise ValueError(f"Could not find a pad token for {model_dir}") + pad_id = padding.get( + "pad_id", + config.get("pad_token_id", 0) if has_config else tokenizer.token_to_id(pad_token), + ) + if pad_id is None: + raise ValueError(f"Could not find pad token {pad_token!r} in {model_dir}") + tokenizer.enable_padding( direction=padding.get("direction", "right"), - pad_id=padding.get("pad_id", config.get("pad_token_id", 0)), + pad_id=pad_id, pad_type_id=padding.get("pad_type_id", 0), pad_token=pad_token, pad_to_multiple_of=padding.get("pad_to_multiple_of"), @@ -108,14 +116,11 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: elif isinstance(token, dict): tokenizer.add_special_tokens([AddedToken(**token)]) - special_token_to_id: dict[str, int] = {} - - for token in tokens_map.values(): - if isinstance(token, str): - special_token_to_id[token] = tokenizer.token_to_id(token) - elif isinstance(token, dict): - token_str = token.get("content", "") - special_token_to_id[token_str] = tokenizer.token_to_id(token_str) + special_token_to_id = { + token.content: token_id + for token_id, token in tokenizer.get_added_tokens_decoder().items() + if token.special + } return tokenizer, special_token_to_id diff --git a/tests/test_preprocessor_utils.py b/tests/test_preprocessor_utils.py index 23cfc58c..1e5a34b9 100644 --- a/tests/test_preprocessor_utils.py +++ b/tests/test_preprocessor_utils.py @@ -68,9 +68,12 @@ def factory( config: dict[str, Any] | None = None, padding: dict[str, Any] | None = None, drop_from_tokenizer_config: tuple[str, ...] = (), + drop_files: tuple[str, ...] = (), ) -> Path: model_dir = tmp_path_factory.mktemp(f"model_dir_{next(counter)}") for file_name in TOKENIZER_FILES: + if file_name in drop_files: + continue shutil.copy(source_dir / file_name, model_dir / file_name) _patch_json( @@ -78,7 +81,8 @@ def factory( tokenizer_config or {}, drop_from_tokenizer_config, ) - _patch_json(model_dir / "config.json", config or {}) + if "config.json" not in drop_files: + _patch_json(model_dir / "config.json", config or {}) if padding is not None: _set_serialized_padding(model_dir / "tokenizer.json", padding) @@ -230,3 +234,32 @@ def test_absent_max_context_keys_raise(make_model_dir) -> None: with pytest.raises(ValueError, match="Could not determine the maximum context length"): load_tokenizer(model_dir) + + +# --- optional tokenizer metadata files (#686) --------------------------------------------- + + +def test_load_tokenizer_without_config(make_model_dir) -> None: + """Without config.json the pad id is resolved from the tokenizer vocabulary.""" + model_dir = make_model_dir(drop_files=("config.json",)) + + tokenizer, special_token_to_id = load_tokenizer(model_dir) + + assert tokenizer.padding["pad_token"] == "[PAD]" + assert tokenizer.padding["pad_id"] == 0 + assert special_token_to_id["[MASK]"] == 103 + + +def test_load_tokenizer_without_special_tokens_map(make_model_dir) -> None: + """Newer transformers releases stop writing the file; the tokenizer already knows them.""" + model_dir = make_model_dir(drop_files=("special_tokens_map.json",)) + + _, special_token_to_id = load_tokenizer(model_dir) + + assert special_token_to_id == { + "[PAD]": 0, + "[UNK]": 100, + "[CLS]": 101, + "[SEP]": 102, + "[MASK]": 103, + } From 57f3620d553ffbce8bdd938422fd2f356edc7db9 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Tue, 22 Sep 2026 00:06:31 +0700 Subject: [PATCH 2/3] fix: pad id fallback chain and additional_special_tokens lists --- fastembed/common/preprocessor_utils.py | 55 +++++--- tests/test_preprocessor_utils.py | 174 +++++++++++++++++++++---- 2 files changed, 189 insertions(+), 40 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 40702b08..693a0c7b 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -1,6 +1,6 @@ import json import sys -from typing import Any +from typing import Any, Iterator from pathlib import Path from tokenizers import AddedToken, Tokenizer @@ -9,6 +9,11 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: + """Read special_tokens_map.json, treating an absent file as an empty map. + + Newer transformers releases stop writing the file, and everything it holds is also + recorded in tokenizer.json, so its absence is not an error. + """ tokens_map_path = model_dir / "special_tokens_map.json" if not tokens_map_path.exists(): return {} @@ -19,6 +24,20 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: return tokens_map +def iter_special_tokens(tokens_map: dict[str, Any]) -> Iterator[str | dict[str, Any]]: + """Yield the individual tokens declared in a special tokens map. + + Most keys hold a single token, either a bare string or an `AddedToken` dict, but + `additional_special_tokens` holds a list of them, which has to be flattened before + the tokens can be dispatched on their type. + """ + for value in tokens_map.values(): + if isinstance(value, list): + yield from value + else: + yield value + + def _valid_context(value: Any) -> int | None: """Return `value` if it can be used as a truncation limit, `None` otherwise. @@ -59,8 +78,6 @@ def _resolve_max_context(tokenizer_config: dict[str, Any], model_dir: Path) -> i def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: - config_path = model_dir / "config.json" - tokenizer_path = model_dir / "tokenizer.json" if not tokenizer_path.exists(): raise ValueError(f"Could not find tokenizer.json in {model_dir}") @@ -69,9 +86,11 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: if not tokenizer_config_path.exists(): raise ValueError(f"Could not find tokenizer_config.json in {model_dir}") - has_config = config_path.exists() + # config.json is optional: it only ever contributes pad_token_id, and newer transformers + # releases no longer write it for every model. + config_path = model_dir / "config.json" config: dict[str, Any] = {} - if has_config: + if config_path.exists(): with open(str(config_path)) as config_file: config = json.load(config_file) @@ -85,6 +104,14 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) + # Special tokens are registered before the padding is resolved: the map may name a pad + # token that tokenizer.json does not carry, and it only gets an id once it is added. + for token in iter_special_tokens(tokens_map): + if isinstance(token, str): + tokenizer.add_special_tokens([token]) + elif isinstance(token, dict): + tokenizer.add_special_tokens([AddedToken(**token)]) + # Padding is always normalized to batch-longest. A serialized fixed length shorter than the # truncation limit leaves longer encodings untouched, which produces ragged batches, and a # fixed length equal to it pads every batch to the maximum. Direction and pad token metadata @@ -94,12 +121,14 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: if pad_token is None: raise ValueError(f"Could not find a pad token for {model_dir}") - pad_id = padding.get( - "pad_id", - config.get("pad_token_id", 0) if has_config else tokenizer.token_to_id(pad_token), - ) + # `config.json` is optional, and even when it is present it does not always carry a + # `pad_token_id`, so the vocabulary is the last resort. A hardcoded 0 is not: it silently + # disagrees with `pad_token` for every model whose pad token is not the first entry. + pad_id = padding.get("pad_id", config.get("pad_token_id")) + if pad_id is None: + pad_id = tokenizer.token_to_id(pad_token) if pad_id is None: - raise ValueError(f"Could not find pad token {pad_token!r} in {model_dir}") + raise ValueError(f"Could not resolve an id for the pad token {pad_token!r} in {model_dir}") tokenizer.enable_padding( direction=padding.get("direction", "right"), @@ -110,12 +139,6 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: length=None, ) - for token in tokens_map.values(): - if isinstance(token, str): - tokenizer.add_special_tokens([token]) - elif isinstance(token, dict): - tokenizer.add_special_tokens([AddedToken(**token)]) - special_token_to_id = { token.content: token_id for token_id, token in tokenizer.get_added_tokens_decoder().items() diff --git a/tests/test_preprocessor_utils.py b/tests/test_preprocessor_utils.py index 1e5a34b9..ce12b7dd 100644 --- a/tests/test_preprocessor_utils.py +++ b/tests/test_preprocessor_utils.py @@ -68,6 +68,7 @@ def factory( config: dict[str, Any] | None = None, padding: dict[str, Any] | None = None, drop_from_tokenizer_config: tuple[str, ...] = (), + drop_from_config: tuple[str, ...] = (), drop_files: tuple[str, ...] = (), ) -> Path: model_dir = tmp_path_factory.mktemp(f"model_dir_{next(counter)}") @@ -76,13 +77,14 @@ def factory( continue shutil.copy(source_dir / file_name, model_dir / file_name) - _patch_json( - model_dir / "tokenizer_config.json", - tokenizer_config or {}, - drop_from_tokenizer_config, - ) + if "tokenizer_config.json" not in drop_files: + _patch_json( + model_dir / "tokenizer_config.json", + tokenizer_config or {}, + drop_from_tokenizer_config, + ) if "config.json" not in drop_files: - _patch_json(model_dir / "config.json", config or {}) + _patch_json(model_dir / "config.json", config or {}, drop_from_config) if padding is not None: _set_serialized_padding(model_dir / "tokenizer.json", padding) @@ -238,28 +240,152 @@ def test_absent_max_context_keys_raise(make_model_dir) -> None: # --- optional tokenizer metadata files (#686) --------------------------------------------- +OPTIONAL_FILES = ("config.json", "special_tokens_map.json") +REQUIRED_FILES = ("tokenizer.json", "tokenizer_config.json") -def test_load_tokenizer_without_config(make_model_dir) -> None: - """Without config.json the pad id is resolved from the tokenizer vocabulary.""" - model_dir = make_model_dir(drop_files=("config.json",)) - tokenizer, special_token_to_id = load_tokenizer(model_dir) +@pytest.fixture(scope="module") +def token_id(make_model_dir): + """Resolve vocabulary ids by name, so the cases below carry no magic numbers.""" + return Tokenizer.from_file(str(make_model_dir() / "tokenizer.json")).token_to_id - assert tokenizer.padding["pad_token"] == "[PAD]" - assert tokenizer.padding["pad_id"] == 0 - assert special_token_to_id["[MASK]"] == 103 +@pytest.mark.parametrize( + "dropped", + [(), ("config.json",), ("special_tokens_map.json",), OPTIONAL_FILES], + ids=["nothing", "no-config", "no-special-tokens-map", "neither"], +) +def test_optional_files_do_not_change_what_is_loaded(make_model_dir, dropped) -> None: + """Both files are redundant: everything they carry is already in the tokenizer. + + Comparing against the full-metadata load, rather than against literal ids, ties the + expectation to that contract instead of to one model's vocabulary. + """ + baseline, baseline_specials = load_tokenizer(make_model_dir()) + + tokenizer, specials = load_tokenizer(make_model_dir(drop_files=dropped)) + + assert specials == baseline_specials + assert tokenizer.padding == baseline.padding + assert tokenizer.encode("hello world").ids == baseline.encode("hello world").ids + + +@pytest.mark.parametrize("missing", REQUIRED_FILES) +def test_the_remaining_files_are_still_required(make_model_dir, missing) -> None: + """Relaxing the optional two must not relax the two that carry irreplaceable data.""" + model_dir = make_model_dir(drop_files=(missing,)) + + with pytest.raises(ValueError, match=f"Could not find {missing}"): + load_tokenizer(model_dir) + + +@pytest.mark.parametrize( + ("model_files", "expected"), + [ + pytest.param( + { + "padding": {"length": None, "pad_id": 7, "pad_token": "[SEP]"}, + "config": {"pad_token_id": 9}, + }, + 7, + id="serialized-padding-wins", + ), + pytest.param( + {"config": {"pad_token_id": 9}, "tokenizer_config": {"pad_token": "[SEP]"}}, + 9, + id="config-beats-the-vocabulary", + ), + pytest.param( + {"tokenizer_config": {"pad_token": "[SEP]"}, "drop_from_config": ("pad_token_id",)}, + "[SEP]", + id="vocabulary-when-config-omits-pad-token-id", + ), + pytest.param( + {"tokenizer_config": {"pad_token": "[SEP]"}, "drop_files": ("config.json",)}, + "[SEP]", + id="vocabulary-when-config-is-absent", + ), + ], +) +def test_pad_id_resolution_order(make_model_dir, token_id, model_files, expected) -> None: + """Serialized padding, then config.json, then the vocabulary. + + A hardcoded 0 is deliberately not the last link: it silently disagrees with `pad_token` + for every model whose pad token is not the first vocabulary entry. `[SEP]` is used + throughout because its id is not 0, so a lookup that never ran cannot pass by accident. + """ + tokenizer, _ = load_tokenizer(make_model_dir(**model_files)) + + assert tokenizer.padding["pad_id"] == ( + token_id(expected) if isinstance(expected, str) else expected + ) + + +def test_pad_token_that_resolves_nowhere_raises(make_model_dir) -> None: + """Without config.json a pad token outside the vocabulary has no id left to fall back on.""" + model_dir = make_model_dir( + tokenizer_config={"pad_token": "[NOT_IN_VOCAB]"}, + drop_files=("config.json",), + ) + + with pytest.raises(ValueError, match="Could not resolve an id for the pad token"): + load_tokenizer(model_dir) + + +def test_tokens_declared_only_in_the_map_are_still_registered(make_model_dir) -> None: + """special_tokens_map.json stops being the source of truth, but must not become a no-op.""" + model_dir = make_model_dir() + _patch_json(model_dir / "special_tokens_map.json", {"sep_token": "<|custom|>"}) + + _, specials = load_tokenizer(model_dir) + + assert "<|custom|>" in specials + + +def test_pad_token_named_only_in_the_map_resolves(make_model_dir) -> None: + """The map is read before the pad id is resolved, so it can name a token tokenizer.json + does not carry. Hand-assembled model directories do exactly this.""" + model_dir = make_model_dir( + tokenizer_config={"pad_token": "<|mypad|>"}, + drop_files=("config.json",), + ) + _patch_json(model_dir / "special_tokens_map.json", {"pad_token": "<|mypad|>"}) + + tokenizer, specials = load_tokenizer(model_dir) + + assert tokenizer.padding["pad_token"] == "<|mypad|>" + assert tokenizer.padding["pad_id"] == specials["<|mypad|>"] -def test_load_tokenizer_without_special_tokens_map(make_model_dir) -> None: - """Newer transformers releases stop writing the file; the tokenizer already knows them.""" - model_dir = make_model_dir(drop_files=("special_tokens_map.json",)) - _, special_token_to_id = load_tokenizer(model_dir) +@pytest.mark.parametrize( + "additional", + [ + pytest.param(["<|list_str|>"], id="list-of-strings"), + pytest.param([{"content": "<|list_str|>"}], id="list-of-added-token-dicts"), + ], +) +def test_list_valued_map_entries_are_registered(make_model_dir, additional) -> None: + """`additional_special_tokens` holds a list, which the str/dict dispatch alone drops. + + Five registry repos ship one, in both spellings. Their tokens happen to be in + tokenizer.json already, so only a token that lives *nowhere else* shows the drop. + """ + model_dir = make_model_dir() + _patch_json(model_dir / "special_tokens_map.json", {"additional_special_tokens": additional}) + + _, specials = load_tokenizer(model_dir) + + assert "<|list_str|>" in specials + + +def test_scalar_map_entries_still_behave(make_model_dir) -> None: + """Flattening lists must not disturb the single-token keys around them.""" + model_dir = make_model_dir() + _patch_json( + model_dir / "special_tokens_map.json", + {"cls_token": "<|scalar_str|>", "additional_special_tokens": ["<|from_list|>"]}, + ) + + _, specials = load_tokenizer(model_dir) - assert special_token_to_id == { - "[PAD]": 0, - "[UNK]": 100, - "[CLS]": 101, - "[SEP]": 102, - "[MASK]": 103, - } + assert {"<|scalar_str|>", "<|from_list|>"} <= specials.keys() From 5fbb5e99e90ce49f2aacda20d8a33629e0f7f9e6 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Tue, 22 Sep 2026 01:19:31 +0700 Subject: [PATCH 3/3] refactor: remove redundant tests and comments --- fastembed/common/preprocessor_utils.py | 22 ++---- tests/test_preprocessor_utils.py | 92 +++++--------------------- 2 files changed, 24 insertions(+), 90 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 693a0c7b..c08658db 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -9,11 +9,7 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: - """Read special_tokens_map.json, treating an absent file as an empty map. - - Newer transformers releases stop writing the file, and everything it holds is also - recorded in tokenizer.json, so its absence is not an error. - """ + """Read special_tokens_map.json, treating an absent file as an empty map.""" tokens_map_path = model_dir / "special_tokens_map.json" if not tokens_map_path.exists(): return {} @@ -27,9 +23,7 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: def iter_special_tokens(tokens_map: dict[str, Any]) -> Iterator[str | dict[str, Any]]: """Yield the individual tokens declared in a special tokens map. - Most keys hold a single token, either a bare string or an `AddedToken` dict, but - `additional_special_tokens` holds a list of them, which has to be flattened before - the tokens can be dispatched on their type. + Most keys hold one token, but `additional_special_tokens` holds a list of them. """ for value in tokens_map.values(): if isinstance(value, list): @@ -86,8 +80,7 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: if not tokenizer_config_path.exists(): raise ValueError(f"Could not find tokenizer_config.json in {model_dir}") - # config.json is optional: it only ever contributes pad_token_id, and newer transformers - # releases no longer write it for every model. + # config.json is optional: transformers v5 no longer writes it for every model. config_path = model_dir / "config.json" config: dict[str, Any] = {} if config_path.exists(): @@ -104,8 +97,8 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) - # Special tokens are registered before the padding is resolved: the map may name a pad - # token that tokenizer.json does not carry, and it only gets an id once it is added. + # Registered before the padding is resolved: the map may name a pad token that + # tokenizer.json does not carry, and it only gets an id once it is added. for token in iter_special_tokens(tokens_map): if isinstance(token, str): tokenizer.add_special_tokens([token]) @@ -121,9 +114,8 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: if pad_token is None: raise ValueError(f"Could not find a pad token for {model_dir}") - # `config.json` is optional, and even when it is present it does not always carry a - # `pad_token_id`, so the vocabulary is the last resort. A hardcoded 0 is not: it silently - # disagrees with `pad_token` for every model whose pad token is not the first entry. + # The vocabulary is the last resort, not a hardcoded 0: that silently disagrees with + # `pad_token` for every model whose pad token is not the first entry. pad_id = padding.get("pad_id", config.get("pad_token_id")) if pad_id is None: pad_id = tokenizer.token_to_id(pad_token) diff --git a/tests/test_preprocessor_utils.py b/tests/test_preprocessor_utils.py index ce12b7dd..0b928f50 100644 --- a/tests/test_preprocessor_utils.py +++ b/tests/test_preprocessor_utils.py @@ -238,12 +238,6 @@ def test_absent_max_context_keys_raise(make_model_dir) -> None: load_tokenizer(model_dir) -# --- optional tokenizer metadata files (#686) --------------------------------------------- - -OPTIONAL_FILES = ("config.json", "special_tokens_map.json") -REQUIRED_FILES = ("tokenizer.json", "tokenizer_config.json") - - @pytest.fixture(scope="module") def token_id(make_model_dir): """Resolve vocabulary ids by name, so the cases below carry no magic numbers.""" @@ -252,15 +246,11 @@ def token_id(make_model_dir): @pytest.mark.parametrize( "dropped", - [(), ("config.json",), ("special_tokens_map.json",), OPTIONAL_FILES], - ids=["nothing", "no-config", "no-special-tokens-map", "neither"], + [("config.json",), ("special_tokens_map.json",), ("config.json", "special_tokens_map.json")], + ids=["no-config", "no-special-tokens-map", "neither"], ) def test_optional_files_do_not_change_what_is_loaded(make_model_dir, dropped) -> None: - """Both files are redundant: everything they carry is already in the tokenizer. - - Comparing against the full-metadata load, rather than against literal ids, ties the - expectation to that contract instead of to one model's vocabulary. - """ + """Both files are redundant: everything they carry is already in the tokenizer.""" baseline, baseline_specials = load_tokenizer(make_model_dir()) tokenizer, specials = load_tokenizer(make_model_dir(drop_files=dropped)) @@ -270,7 +260,7 @@ def test_optional_files_do_not_change_what_is_loaded(make_model_dir, dropped) -> assert tokenizer.encode("hello world").ids == baseline.encode("hello world").ids -@pytest.mark.parametrize("missing", REQUIRED_FILES) +@pytest.mark.parametrize("missing", ("tokenizer.json", "tokenizer_config.json")) def test_the_remaining_files_are_still_required(make_model_dir, missing) -> None: """Relaxing the optional two must not relax the two that carry irreplaceable data.""" model_dir = make_model_dir(drop_files=(missing,)) @@ -280,45 +270,21 @@ def test_the_remaining_files_are_still_required(make_model_dir, missing) -> None @pytest.mark.parametrize( - ("model_files", "expected"), + "model_files", [ - pytest.param( - { - "padding": {"length": None, "pad_id": 7, "pad_token": "[SEP]"}, - "config": {"pad_token_id": 9}, - }, - 7, - id="serialized-padding-wins", - ), - pytest.param( - {"config": {"pad_token_id": 9}, "tokenizer_config": {"pad_token": "[SEP]"}}, - 9, - id="config-beats-the-vocabulary", - ), - pytest.param( - {"tokenizer_config": {"pad_token": "[SEP]"}, "drop_from_config": ("pad_token_id",)}, - "[SEP]", - id="vocabulary-when-config-omits-pad-token-id", - ), - pytest.param( - {"tokenizer_config": {"pad_token": "[SEP]"}, "drop_files": ("config.json",)}, - "[SEP]", - id="vocabulary-when-config-is-absent", - ), + pytest.param({"drop_from_config": ("pad_token_id",)}, id="config-omits-pad-token-id"), + pytest.param({"drop_files": ("config.json",)}, id="config-is-absent"), ], ) -def test_pad_id_resolution_order(make_model_dir, token_id, model_files, expected) -> None: - """Serialized padding, then config.json, then the vocabulary. +def test_pad_id_falls_back_to_the_vocabulary(make_model_dir, token_id, model_files) -> None: + """Last link of the chain; a hardcoded 0 would silently disagree with `pad_token`.""" + expected = token_id("[SEP]") + assert expected != 0, "a pad token whose id is 0 would pass even without a lookup" + model_dir = make_model_dir(tokenizer_config={"pad_token": "[SEP]"}, **model_files) - A hardcoded 0 is deliberately not the last link: it silently disagrees with `pad_token` - for every model whose pad token is not the first vocabulary entry. `[SEP]` is used - throughout because its id is not 0, so a lookup that never ran cannot pass by accident. - """ - tokenizer, _ = load_tokenizer(make_model_dir(**model_files)) + tokenizer, _ = load_tokenizer(model_dir) - assert tokenizer.padding["pad_id"] == ( - token_id(expected) if isinstance(expected, str) else expected - ) + assert tokenizer.padding["pad_id"] == expected def test_pad_token_that_resolves_nowhere_raises(make_model_dir) -> None: @@ -332,19 +298,8 @@ def test_pad_token_that_resolves_nowhere_raises(make_model_dir) -> None: load_tokenizer(model_dir) -def test_tokens_declared_only_in_the_map_are_still_registered(make_model_dir) -> None: - """special_tokens_map.json stops being the source of truth, but must not become a no-op.""" - model_dir = make_model_dir() - _patch_json(model_dir / "special_tokens_map.json", {"sep_token": "<|custom|>"}) - - _, specials = load_tokenizer(model_dir) - - assert "<|custom|>" in specials - - def test_pad_token_named_only_in_the_map_resolves(make_model_dir) -> None: - """The map is read before the pad id is resolved, so it can name a token tokenizer.json - does not carry. Hand-assembled model directories do exactly this.""" + """The map is read first, so it can name a pad token tokenizer.json does not carry.""" model_dir = make_model_dir( tokenizer_config={"pad_token": "<|mypad|>"}, drop_files=("config.json",), @@ -367,8 +322,8 @@ def test_pad_token_named_only_in_the_map_resolves(make_model_dir) -> None: def test_list_valued_map_entries_are_registered(make_model_dir, additional) -> None: """`additional_special_tokens` holds a list, which the str/dict dispatch alone drops. - Five registry repos ship one, in both spellings. Their tokens happen to be in - tokenizer.json already, so only a token that lives *nowhere else* shows the drop. + Real repos ship both spellings, and their tokens are in tokenizer.json already, so + only a token living nowhere else shows the drop. """ model_dir = make_model_dir() _patch_json(model_dir / "special_tokens_map.json", {"additional_special_tokens": additional}) @@ -376,16 +331,3 @@ def test_list_valued_map_entries_are_registered(make_model_dir, additional) -> N _, specials = load_tokenizer(model_dir) assert "<|list_str|>" in specials - - -def test_scalar_map_entries_still_behave(make_model_dir) -> None: - """Flattening lists must not disturb the single-token keys around them.""" - model_dir = make_model_dir() - _patch_json( - model_dir / "special_tokens_map.json", - {"cls_token": "<|scalar_str|>", "additional_special_tokens": ["<|from_list|>"]}, - ) - - _, specials = load_tokenizer(model_dir) - - assert {"<|scalar_str|>", "<|from_list|>"} <= specials.keys()