From 6f35a0a31a254716476e530fc6eda316ec1f667f Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:40:54 +0200 Subject: [PATCH 1/7] Fix padding configuration in preprocessor_utils.py --- fastembed/common/preprocessor_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 3b702f79..e899700e 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -51,8 +51,19 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) if not tokenizer.padding: + pad_to_multiple_of = ( + tokenizer_config.get("pad_to_multiple_of") + if tokenizer_config.get("pad_to_multiple_of") is not None + else ( + config.get("pad_to_multiple_of") + if config.get("pad_to_multiple_of") is not None + else tokenizer_config.get("pad_to_multiple_of") + ) + ) tokenizer.enable_padding( - pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"] + pad_id=config.get("pad_token_id", 0), + pad_token=tokenizer_config["pad_token"], + pad_to_multiple_of=pad_to_multiple_of, ) for token in tokens_map.values(): From e6db57a87d54266907e3ace9dc367561e8f4c6c5 Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:42:09 +0200 Subject: [PATCH 2/7] Add test for loading tokenizer with pad_to_multiple_of Added a test to verify loading of tokenizer with padding configuration. --- tests/test_common.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_common.py b/tests/test_common.py index f7cae5ba..0b8bd98d 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -59,3 +59,28 @@ def test_last_token_pooling_with_left_padding(): pooled = last_token_pooling(token_embeddings, attention_mask) assert np.allclose(pooled, [[2.0, 2.0], [6.0, 6.0]]) + + +def test_load_tokenizer_with_pad_to_multiple_of(tmp_path): + import json + from fastembed.common.preprocessor_utils import load_tokenizer + + model_dir = tmp_path / "model" + model_dir.mkdir() + + (model_dir / "config.json").write_text(json.dumps({"pad_token_id": 0})) + (model_dir / "tokenizer_config.json").write_text( + json.dumps({"model_max_length": 128, "pad_token": "", "pad_to_multiple_of": 8}) + ) + (model_dir / "special_tokens_map.json").write_text(json.dumps({})) + + from tokenizers import Tokenizer + from tokenizers.models import BPE + + tokenizer = Tokenizer(BPE()) + tokenizer.save(str(model_dir / "tokenizer.json")) + + loaded_tokenizer, _ = load_tokenizer(model_dir) + + assert loaded_tokenizer.padding is not None + assert loaded_tokenizer.padding.get("pad_to_multiple_of") == 8 From 498681bb853a2029f0f67ba790c475f41d85fcbc Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:45:39 +0200 Subject: [PATCH 3/7] Improve tokenizer padding handling Refactor padding logic in tokenizer configuration. --- fastembed/common/preprocessor_utils.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index e899700e..2a414c63 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -1,6 +1,6 @@ import json -from typing import Any from pathlib import Path +from typing import Any from tokenizers import AddedToken, Tokenizer @@ -51,19 +51,18 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) if not tokenizer.padding: - pad_to_multiple_of = ( - tokenizer_config.get("pad_to_multiple_of") - if tokenizer_config.get("pad_to_multiple_of") is not None - else ( - config.get("pad_to_multiple_of") - if config.get("pad_to_multiple_of") is not None - else tokenizer_config.get("pad_to_multiple_of") - ) + tokenizer.enable_padding( + pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"] ) + elif tokenizer.padding.get("length") is not None: + padding_params = tokenizer.padding tokenizer.enable_padding( - pad_id=config.get("pad_token_id", 0), - pad_token=tokenizer_config["pad_token"], - pad_to_multiple_of=pad_to_multiple_of, + direction=padding_params.get("direction", "right"), + pad_id=padding_params.get("pad_id", config.get("pad_token_id", 0)), + pad_type_id=padding_params.get("pad_type_id", 0), + pad_token=padding_params.get("pad_token", tokenizer_config.get("pad_token", "[PAD]")), + pad_to_multiple_of=padding_params.get("pad_to_multiple_of"), + length=None, ) for token in tokens_map.values(): From 6fb5e843a0ac6b137bb664ce38feb5fca8c6f468 Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:46:08 +0200 Subject: [PATCH 4/7] Improve tokenizer test cases and assertions Refactor tokenizer tests for clarity and correctness. --- tests/test_common.py | 93 +++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 0b8bd98d..828e4af8 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,11 +1,11 @@ import numpy as np from fastembed import ( - TextEmbedding, - SparseTextEmbedding, ImageEmbedding, LateInteractionMultimodalEmbedding, LateInteractionTextEmbedding, + SparseTextEmbedding, + TextEmbedding, ) from fastembed.common.utils import last_token_pooling @@ -23,13 +23,13 @@ def test_text_list_supported_models(): description = supported_models[0] assert isinstance(description, dict) - assert "model" in description and description["model"] + assert description.get("model") if model_type != SparseTextEmbedding: - assert "dim" in description and description["dim"] - assert "license" in description and description["license"] - assert "size_in_GB" in description and description["size_in_GB"] - assert "model_file" in description and description["model_file"] - assert "sources" in description and description["sources"] + assert description.get("dim") + assert description.get("license") + assert description.get("size_in_GB") + assert description.get("model_file") + assert description.get("sources") assert "hf" in description["sources"] or "url" in description["sources"] @@ -61,26 +61,75 @@ def test_last_token_pooling_with_left_padding(): assert np.allclose(pooled, [[2.0, 2.0], [6.0, 6.0]]) -def test_load_tokenizer_with_pad_to_multiple_of(tmp_path): +def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): import json + + from tokenizers import Tokenizer, models + from fastembed.common.preprocessor_utils import load_tokenizer - model_dir = tmp_path / "model" - model_dir.mkdir() + config = {"pad_token_id": 0} + with open(tmp_path / "config.json", "w") as f: + json.dump(config, f) - (model_dir / "config.json").write_text(json.dumps({"pad_token_id": 0})) - (model_dir / "tokenizer_config.json").write_text( - json.dumps({"model_max_length": 128, "pad_token": "", "pad_to_multiple_of": 8}) - ) - (model_dir / "special_tokens_map.json").write_text(json.dumps({})) + tokenizer_config = { + "model_max_length": 512, + "pad_token": "[PAD]", + } + with open(tmp_path / "tokenizer_config.json", "w") as f: + json.dump(tokenizer_config, f) + + with open(tmp_path / "special_tokens_map.json", "w") as f: + json.dump({"pad_token": "[PAD]"}, f) + + # Tokenizer initialized with fixed-length padding (e.g. gte-base with length=128) + tokenizer = Tokenizer(models.BPE()) + tokenizer.add_special_tokens(["[PAD]"]) + tokenizer.enable_padding(length=128, pad_id=0, pad_token="[PAD]", direction="right") + tokenizer.save(str(tmp_path / "tokenizer.json")) + + loaded_tokenizer, _ = load_tokenizer(tmp_path) + + # Fixed length must be relaxed to None (dynamic batch padding) to prevent ragged arrays + assert loaded_tokenizer.padding is not None + assert loaded_tokenizer.padding["length"] is None + assert loaded_tokenizer.padding["direction"] == "right" + assert loaded_tokenizer.padding["pad_id"] == 0 + assert loaded_tokenizer.padding["pad_token"] == "[PAD]" + assert loaded_tokenizer.truncation["max_length"] == 512 + + +def test_load_tokenizer_preserves_left_padding(tmp_path): + import json + + from tokenizers import Tokenizer, models + + from fastembed.common.preprocessor_utils import load_tokenizer + + config = {"pad_token_id": 50283} + with open(tmp_path / "config.json", "w") as f: + json.dump(config, f) + + tokenizer_config = { + "model_max_length": 8192, + "pad_token": "[PAD]", + } + with open(tmp_path / "tokenizer_config.json", "w") as f: + json.dump(tokenizer_config, f) - from tokenizers import Tokenizer - from tokenizers.models import BPE + with open(tmp_path / "special_tokens_map.json", "w") as f: + json.dump({"pad_token": "[PAD]"}, f) - tokenizer = Tokenizer(BPE()) - tokenizer.save(str(model_dir / "tokenizer.json")) + # Tokenizer with dynamic left padding (e.g. ColModernVBERT) + tokenizer = Tokenizer(models.BPE()) + tokenizer.add_special_tokens(["[PAD]"]) + tokenizer.enable_padding(length=None, pad_id=50283, pad_token="[PAD]", direction="left") + tokenizer.save(str(tmp_path / "tokenizer.json")) - loaded_tokenizer, _ = load_tokenizer(model_dir) + loaded_tokenizer, _ = load_tokenizer(tmp_path) + # Preserves left padding direction and pad_id assert loaded_tokenizer.padding is not None - assert loaded_tokenizer.padding.get("pad_to_multiple_of") == 8 + assert loaded_tokenizer.padding["length"] is None + assert loaded_tokenizer.padding["direction"] == "left" + assert loaded_tokenizer.padding["pad_id"] == 50283 From 6bb4944d840abab6effce9880821c5f7a5012ec3 Mon Sep 17 00:00:00 2001 From: Mohammed Alshek Date: Sun, 6 Sep 2026 01:03:46 +0200 Subject: [PATCH 5/7] fix: address code review feedback on padding handling and coverage - Add docstrings to satisfy documentation coverage threshold - Validate pad_to_multiple_of as a positive integer - Support config.json fallback for pad_to_multiple_of - Apply configured pad_to_multiple_of to serialized padding settings - Add comprehensive unit tests covering validation and precedence --- fastembed/common/preprocessor_utils.py | 69 +++++++++++++++++++++---- tests/test_common.py | 71 ++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 2a414c63..dedb7e0a 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -19,6 +19,25 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: + """ + Load and configure a tokenizer from a model directory. + + Configures truncation to the model context length, converts any fixed-length + padding to dynamic batch padding (avoiding ragged batch failures), preserves + serialized padding direction and token IDs, and optionally applies padding + multiples (e.g. pad_to_multiple_of) when configured. + + Args: + model_dir: Directory path containing tokenizer configuration files + (config.json, tokenizer.json, tokenizer_config.json, special_tokens_map.json). + + Returns: + A tuple of (configured Tokenizer instance, mapping of special token strings to token IDs). + + Raises: + ValueError: If required configuration files are missing or if pad_to_multiple_of + is not a positive integer. + """ config_path = model_dir / "config.json" if not config_path.exists(): raise ValueError(f"Could not find config.json in {model_dir}") @@ -50,20 +69,52 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) + + pad_to_multiple_of = tokenizer_config.get("pad_to_multiple_of") + if pad_to_multiple_of is None: + pad_to_multiple_of = config.get("pad_to_multiple_of") + + if pad_to_multiple_of is not None and ( + not isinstance(pad_to_multiple_of, int) + or isinstance(pad_to_multiple_of, bool) + or pad_to_multiple_of <= 0 + ): + raise ValueError("pad_to_multiple_of must be a positive integer") + if not tokenizer.padding: tokenizer.enable_padding( - pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"] + pad_id=config.get("pad_token_id", 0), + pad_token=tokenizer_config.get("pad_token", "[PAD]"), + pad_to_multiple_of=pad_to_multiple_of, ) - elif tokenizer.padding.get("length") is not None: + else: padding_params = tokenizer.padding - tokenizer.enable_padding( - direction=padding_params.get("direction", "right"), - pad_id=padding_params.get("pad_id", config.get("pad_token_id", 0)), - pad_type_id=padding_params.get("pad_type_id", 0), - pad_token=padding_params.get("pad_token", tokenizer_config.get("pad_token", "[PAD]")), - pad_to_multiple_of=padding_params.get("pad_to_multiple_of"), - length=None, + target_pad_to_multiple_of = ( + pad_to_multiple_of + if pad_to_multiple_of is not None + else padding_params.get("pad_to_multiple_of") ) + if target_pad_to_multiple_of is not None and ( + not isinstance(target_pad_to_multiple_of, int) + or isinstance(target_pad_to_multiple_of, bool) + or target_pad_to_multiple_of <= 0 + ): + raise ValueError("pad_to_multiple_of must be a positive integer") + + if padding_params.get("length") is not None or ( + pad_to_multiple_of is not None + and padding_params.get("pad_to_multiple_of") != target_pad_to_multiple_of + ): + tokenizer.enable_padding( + direction=padding_params.get("direction", "right"), + pad_id=padding_params.get("pad_id", config.get("pad_token_id", 0)), + pad_type_id=padding_params.get("pad_type_id", 0), + pad_token=padding_params.get( + "pad_token", tokenizer_config.get("pad_token", "[PAD]") + ), + pad_to_multiple_of=target_pad_to_multiple_of, + length=None, + ) for token in tokens_map.values(): if isinstance(token, str): diff --git a/tests/test_common.py b/tests/test_common.py index 828e4af8..744219fa 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -62,6 +62,10 @@ def test_last_token_pooling_with_left_padding(): def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): + """ + Verify that models with serialized fixed-length padding (e.g. gte-base with length=128) + have their padding relaxed to dynamic batch padding (length=None) to support mixed-length batches. + """ import json from tokenizers import Tokenizer, models @@ -100,6 +104,10 @@ def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): def test_load_tokenizer_preserves_left_padding(tmp_path): + """ + Verify that tokenizers with serialized left padding (e.g. ColModernVBERT) + preserve their padding direction and pad_token_id without being overridden. + """ import json from tokenizers import Tokenizer, models @@ -133,3 +141,66 @@ def test_load_tokenizer_preserves_left_padding(tmp_path): assert loaded_tokenizer.padding["length"] is None assert loaded_tokenizer.padding["direction"] == "left" assert loaded_tokenizer.padding["pad_id"] == 50283 + + +def test_load_tokenizer_pad_to_multiple_of_and_validation(tmp_path): + """ + Verify pad_to_multiple_of configuration precedence (tokenizer_config > config fallback), + application to unpadded and serialized tokenizers, and validation of positive integer values. + """ + import json + + import pytest + from tokenizers import Tokenizer, models + + from fastembed.common.preprocessor_utils import load_tokenizer + + # 1. Test pad_to_multiple_of via config.json fallback + dir_fallback = tmp_path / "fallback" + dir_fallback.mkdir() + (dir_fallback / "config.json").write_text( + json.dumps({"pad_token_id": 0, "pad_to_multiple_of": 16}) + ) + (dir_fallback / "tokenizer_config.json").write_text( + json.dumps({"model_max_length": 128, "pad_token": "[PAD]"}) + ) + (dir_fallback / "special_tokens_map.json").write_text(json.dumps({"pad_token": "[PAD]"})) + tok = Tokenizer(models.BPE()) + tok.add_special_tokens(["[PAD]"]) + tok.save(str(dir_fallback / "tokenizer.json")) + + loaded_tok, _ = load_tokenizer(dir_fallback) + assert loaded_tok.padding is not None + assert loaded_tok.padding.get("pad_to_multiple_of") == 16 + + # 2. Test tokenizer_config.json precedence over config.json + dir_prec = tmp_path / "prec" + dir_prec.mkdir() + (dir_prec / "config.json").write_text( + json.dumps({"pad_token_id": 0, "pad_to_multiple_of": 16}) + ) + (dir_prec / "tokenizer_config.json").write_text( + json.dumps({"model_max_length": 128, "pad_token": "[PAD]", "pad_to_multiple_of": 8}) + ) + (dir_prec / "special_tokens_map.json").write_text(json.dumps({"pad_token": "[PAD]"})) + tok.save(str(dir_prec / "tokenizer.json")) + + loaded_tok, _ = load_tokenizer(dir_prec) + assert loaded_tok.padding is not None + assert loaded_tok.padding.get("pad_to_multiple_of") == 8 + + # 3. Test validation error on invalid pad_to_multiple_of (<= 0 or non-integer) + for invalid_val in [0, -1, "8", False]: + dir_invalid = tmp_path / f"invalid_{invalid_val}" + dir_invalid.mkdir() + (dir_invalid / "config.json").write_text(json.dumps({"pad_token_id": 0})) + (dir_invalid / "tokenizer_config.json").write_text( + json.dumps( + {"model_max_length": 128, "pad_token": "[PAD]", "pad_to_multiple_of": invalid_val} + ) + ) + (dir_invalid / "special_tokens_map.json").write_text(json.dumps({"pad_token": "[PAD]"})) + tok.save(str(dir_invalid / "tokenizer.json")) + + with pytest.raises(ValueError, match="pad_to_multiple_of must be a positive integer"): + load_tokenizer(dir_invalid) From bf7b67625e1eee5548362beb585ce1818701fa2c Mon Sep 17 00:00:00 2001 From: Mohammed Alshyakh Date: Fri, 18 Sep 2026 18:22:11 +0200 Subject: [PATCH 6/7] docs(tests): add docstrings to satisfy pre-merge coverage threshold --- tests/test_common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_common.py b/tests/test_common.py index 744219fa..c88e92ab 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -11,6 +11,7 @@ def test_text_list_supported_models(): + """Verify that supported text models expose non-empty metadata fields.""" for model_type in [ TextEmbedding, SparseTextEmbedding, @@ -34,6 +35,7 @@ def test_text_list_supported_models(): def test_last_token_pooling(): + """Verify last token pooling logic against standard right-padded inputs.""" token_embeddings = np.array( [ [[1.0, 1.0], [2.0, 2.0], [9.0, 9.0], [9.0, 9.0]], # 2 real tokens, then padding @@ -48,6 +50,7 @@ def test_last_token_pooling(): def test_last_token_pooling_with_left_padding(): + """Verify last token pooling logic correctly identifies targets under left padding.""" token_embeddings = np.array( [ [[9.0, 9.0], [9.0, 9.0], [1.0, 1.0], [2.0, 2.0]], # padding, then 2 real tokens From 318195cc1d98a763b0add9ed72359d4be7fbda44 Mon Sep 17 00:00:00 2001 From: Mohammed Alshyakh Date: Fri, 18 Sep 2026 18:45:56 +0200 Subject: [PATCH 7/7] test: assert rectangular batch shape on dynamic-padded tokenizer --- tests/test_common.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index c88e92ab..146cd9e5 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -71,7 +71,8 @@ def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): """ import json - from tokenizers import Tokenizer, models + import numpy as np + from tokenizers import Tokenizer, models, pre_tokenizers from fastembed.common.preprocessor_utils import load_tokenizer @@ -90,7 +91,8 @@ def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): json.dump({"pad_token": "[PAD]"}, f) # Tokenizer initialized with fixed-length padding (e.g. gte-base with length=128) - tokenizer = Tokenizer(models.BPE()) + tokenizer = Tokenizer(models.WordLevel({"[PAD]": 0, "first": 1, "second": 2, "third": 3}, unk_token="[PAD]")) + tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() tokenizer.add_special_tokens(["[PAD]"]) tokenizer.enable_padding(length=128, pad_id=0, pad_token="[PAD]", direction="right") tokenizer.save(str(tmp_path / "tokenizer.json")) @@ -105,6 +107,16 @@ def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): assert loaded_tokenizer.padding["pad_token"] == "[PAD]" assert loaded_tokenizer.truncation["max_length"] == 512 + # Verify that mixed-length batch inputs produce rectangular arrays rather than ragged batches + encodings = loaded_tokenizer.encode_batch(["first", "second third"]) + ids = np.array([e.ids for e in encodings]) + attention_mask = np.array([e.attention_mask for e in encodings]) + + assert ids.shape == (2, 2) + assert attention_mask.shape == (2, 2) + assert np.array_equal(ids, [[1, 0], [2, 3]]) + assert np.array_equal(attention_mask, [[1, 0], [1, 1]]) + def test_load_tokenizer_preserves_left_padding(tmp_path): """