diff --git a/README.md b/README.md index c01586246..7e8d90563 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ ## Latest News 🗞️🚀 +* 08/25/2026 7.4.0-dev `main`: ✨ Added Tencent `HunyuanOCR` quantization support. * 08/25/2026 7.4.0-dev `main`: ✨ Added `lm_head` and embedding quantization lifecycle. * 08/24/2026 7.4.0-dev `main`: ✨ Added Baidu `Unlimited-OCR` quantization support. * 08/20/2026 7.4.0-dev `main`: ✨ Added `deepseek_v32` / DeepSeek V3.2; `muse_glimmer` / Muse Glimmer multimodal; `mage_vl` / Mage-VL; Cohere `North Micro Vision` (`cohere_compass`); `axk2` (A.X-K2) model support. @@ -269,7 +270,7 @@ Selected public references where teams or companies explicitly mention GPT-QMode | MiniMax M2/M3 | ✅ | AfMoE | ✅ | Bailing-MoE | ✅ | LFM2 / LFM2-VL / LFM2-MoE | ✅ | Marin | ✅ | | InternVL Chat | ✅ | Laguna | ✅ | Mimo / Mimo V2 | ✅ | Zamba / Zamba2 | ✅ | Intern S1 / S2 Preview | ✅ | | HunYuan V1 Dense / MoE | ✅ | HY-V3 | ✅ | Inkling | ✅ | Solar Open / Open 2 | ✅ | North Micro Vision | ✅ | -| Mage-VL | ✅ | Unlimited-OCR | ✅ | | | | | | | +| Mage-VL | ✅ | Unlimited-OCR | ✅ | HunyuanOCR | ✅ | | | | | | Muse Glimmer | ✅ | | | | | | | | | | SmolLM3 | ✅ | | | | | | | | | diff --git a/gptqmodel/models/auto.py b/gptqmodel/models/auto.py index 39d9520c5..4d493e372 100644 --- a/gptqmodel/models/auto.py +++ b/gptqmodel/models/auto.py @@ -118,6 +118,7 @@ from .definitions.hrm_text import HrmTextQModel # noqa: E402 from .definitions.hunyuan_v1_dense import HunYuanDenseV1QModel # noqa: E402 from .definitions.hunyuan_v1_moe import HunYuanMoEV1QModel # noqa: E402 +from .definitions.hunyuan_vl import HunYuanVLQModel # noqa: E402 from .definitions.hy_v3 import HYV3QModel # noqa: E402 from .definitions.hymba import HymbaQModel # noqa: E402 from .definitions.instella import InstellaQModel # noqa: E402 @@ -257,6 +258,7 @@ "hrm_text": HrmTextQModel, "hunyuan_v1_dense": HunYuanDenseV1QModel, "hunyuan_v1_moe": HunYuanMoEV1QModel, + "hunyuan_vl": HunYuanVLQModel, "hy_v3": HYV3QModel, "qwen": QwenQModel, "mistral": LlamaQModel, # 100% llama clone diff --git a/gptqmodel/models/definitions/__init__.py b/gptqmodel/models/definitions/__init__.py index 625eacb3c..efaf73b7f 100644 --- a/gptqmodel/models/definitions/__init__.py +++ b/gptqmodel/models/definitions/__init__.py @@ -50,6 +50,7 @@ from .hrm_text import HrmTextQModel from .hunyuan_v1_dense import HunYuanDenseV1QModel from .hunyuan_v1_moe import HunYuanMoEV1QModel +from .hunyuan_vl import HunYuanVLQModel from .hy_v3 import HYV3QModel from .hymba import HymbaQModel from .instella import InstellaQModel diff --git a/gptqmodel/models/definitions/hunyuan_vl.py b/gptqmodel/models/definitions/hunyuan_vl.py new file mode 100644 index 000000000..2c0f95cb6 --- /dev/null +++ b/gptqmodel/models/definitions/hunyuan_vl.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from typing import Any, Dict + +from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin + +from ...utils.calibration import batched +from ...utils.looper_helpers import normalize_device_like +from ...utils.model import MODALITY, move_to +from ...utils.offload import offload_to_disk +from .._const import CPU +from ..base import BaseQModel + + +class HunYuanVLQModel(BaseQModel): + """Quantization definition for native Transformers Hunyuan-VL checkpoints.""" + + loader = AutoModelForImageTextToText + + require_load_processor = True + + modality = [MODALITY.TEXT, MODALITY.IMAGE_TO_TEXT] + + pre_lm_head_norm_module = "model.language_model.norm" + rotary_embedding = "model.language_model.rotary_emb" + + module_tree = [ + "model", + "language_model", + "layers", + "#", + { + "input_layernorm": ("input_layernorm:!",), + "self_attn": ( + "query_layernorm:!", + "q_proj:0", + "key_layernorm:!", + "k_proj:0", + "v_proj:0", + "o_proj:1", + ), + "post_attention_layernorm": ("post_attention_layernorm:!",), + "mlp": ("gate_proj:0", "up_proj:0", "down_proj:1"), + }, + ] + + def _materialize_module(self, parent, name: str, module_path: str): + module = getattr(parent, name) + target_device = normalize_device_like(self.quantize_config.device) or CPU + setattr( + parent, + name, + self.shell_module_materialize( + module, + target_device, + module_path=module_path, + ), + ) + + def pre_quantize_generate_hook_start(self): + core_model = self.model.model + language_model = core_model.language_model + self._materialize_module( + language_model, "embed_tokens", "model.language_model.embed_tokens" + ) + self._materialize_module(language_model, "norm", "model.language_model.norm") + self._materialize_module( + language_model, "rotary_emb", "model.language_model.rotary_emb" + ) + self._materialize_module(core_model, "vision_tower", "model.vision_tower") + + def pre_quantize_generate_hook_end(self): + core_model = self.model.model + language_model = core_model.language_model + modules = ( + (language_model, "embed_tokens"), + (language_model, "norm"), + (language_model, "rotary_emb"), + (core_model, "vision_tower"), + ) + + if self.quantize_config.offload_to_disk: + for parent, name in modules: + offload_to_disk( + model=parent, + module=getattr(parent, name), + disk_path=self.quantize_config.offload_to_disk_path, + ) + return + + for parent, name in modules: + setattr(parent, name, move_to(getattr(parent, name), device=CPU)) + + def preprocess_dataset(self, sample: Dict) -> Dict: + return sample + + def load_processor(self) -> ProcessorMixin: + return AutoProcessor.from_pretrained( + self.model_local_path, + trust_remote_code=False, + backend="pil", + ) + + def prepare_dataset( + self, + calibration_dataset, + batch_size: int = 1, + **kwargs, + ) -> list[Dict[str, Any]]: + del kwargs + processor = self.load_processor() + calibration_data = [] + for batch in batched( + calibration_dataset, + batch_size, + process_func=self.preprocess_dataset, + ): + inputs = processor.apply_chat_template( + batch, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + processor_kwargs={"padding": True}, + ) + calibration_data.append(inputs) + del processor + return calibration_data + + +__all__ = ["HunYuanVLQModel"] diff --git a/gptqmodel/utils/hf.py b/gptqmodel/utils/hf.py index f6c47c0e8..3ccd75fab 100644 --- a/gptqmodel/utils/hf.py +++ b/gptqmodel/utils/hf.py @@ -1209,6 +1209,17 @@ def _normalize_chatglm_remote_code_config_compat(config: Any) -> None: def _normalize_rope_parameters_config_compat(config: Any) -> None: + # HunYuanVL is a composite config: RoPE belongs exclusively to its text + # sub-config. Adding a synthetic top-level rope_parameters value is not + # harmless here because HunYuanVLConfig treats flat text fields as legacy + # overrides when the config is serialized and loaded again. That would + # replace the nested multimodal mrope_section with a default RoPE config. + if getattr(config, "model_type", None) == "hunyuan_vl": + text_config = getattr(config, "text_config", None) + if text_config is not None: + _normalize_rope_parameters_config_compat(text_config) + return + rope_parameters = getattr(config, "rope_parameters", None) if ( isinstance(rope_parameters, dict) diff --git a/tests/models/ovis/image_to_test_dataset.py b/tests/models/ovis/image_to_test_dataset.py index 1ead4f06d..8e0ef4e4b 100644 --- a/tests/models/ovis/image_to_test_dataset.py +++ b/tests/models/ovis/image_to_test_dataset.py @@ -9,6 +9,7 @@ from gptqmodel.models.definitions.deepseek_vl import DeepSeekVLQModel from gptqmodel.models.definitions.deepseek_vl_v2 import DeepSeekVLV2QModel from gptqmodel.models.definitions.ernie4_5_vl_moe import Ernie4_5_VLMoeQModel +from gptqmodel.models.definitions.hunyuan_vl import HunYuanVLQModel from gptqmodel.models.definitions.intern_s2_preview import InternS2PreviewQModel from gptqmodel.models.definitions.inkling import InklingMMQModel from gptqmodel.models.definitions.interns1 import InternS1QModel @@ -112,6 +113,25 @@ def format_unlimited_ocr_dataset(image, assistant): } +def format_hunyuan_ocr_dataset(image, assistant): + del assistant + return [ + { + "role": "user", + "content": [ + {"type": "image", "image": image}, + { + "type": "text", + "text": ( + "提取文档图片中正文的所有信息用markdown格式表示，其中页眉、页脚部分忽略，" + "表格用html格式表达，文档中公式用latex格式表示，按照阅读顺序组织进行解析。" + ), + }, + ], + } + ] + + def format_qwen2_5_omni_dataset(image, assistant): return [ { @@ -160,6 +180,10 @@ def prepare_unlimited_ocr_dataset(n_sample: int = 20) -> list[dict]: return prepare_dataset(format_unlimited_ocr_dataset, n_sample=n_sample) +def prepare_hunyuan_ocr_dataset(n_sample: int = 20) -> list[list[dict]]: + return prepare_dataset(format_hunyuan_ocr_dataset, n_sample=n_sample) + + def get_calib_dataset(model): if isinstance(model, OvisQModel): return prepare_dataset(format_ovis_dataset, n_sample=20) @@ -204,4 +228,7 @@ def get_calib_dataset(model): if isinstance(model, UnlimitedOCRQModel): return prepare_unlimited_ocr_dataset(n_sample=20) + if isinstance(model, HunYuanVLQModel): + return prepare_hunyuan_ocr_dataset(n_sample=20) + raise NotImplementedError(f"Unsupported MODEL: {model.__class__}") diff --git a/tests/models/test_hunyuan_ocr.py b/tests/models/test_hunyuan_ocr.py new file mode 100644 index 000000000..9979215ef --- /dev/null +++ b/tests/models/test_hunyuan_ocr.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2026 ModelCloud.ai +# SPDX-FileCopyrightText: 2026 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from gptqmodel import BACKEND +from model_test import ModelTest +from ovis import image_to_test_dataset + + +def test_prepare_hunyuan_ocr_dataset_reuses_shared_dataset(monkeypatch): + calls = {} + + def fake_prepare_dataset(format_func, n_sample): + calls["format_func"] = format_func + calls["n_sample"] = n_sample + return [format_func("image-url", "caption")] + + monkeypatch.setattr(image_to_test_dataset, "prepare_dataset", fake_prepare_dataset) + + dataset = image_to_test_dataset.prepare_hunyuan_ocr_dataset(n_sample=3) + + assert calls == { + "format_func": image_to_test_dataset.format_hunyuan_ocr_dataset, + "n_sample": 3, + } + assert dataset == [ + [ + { + "role": "user", + "content": [ + {"type": "image", "image": "image-url"}, + { + "type": "text", + "text": ( + "提取文档图片中正文的所有信息用markdown格式表示，其中页眉、页脚部分忽略，" + "表格用html格式表达，文档中公式用latex格式表示，按照阅读顺序组织进行解析。" + ), + }, + ], + } + ] + ] + + +class TestHunyuanOCR(ModelTest): + NATIVE_MODEL_ID = "tencent/HunyuanOCR" + TRUST_REMOTE_CODE = False + USE_FLASH_ATTN = False + LOAD_BACKEND = BACKEND.AUTO + EVAL_BATCH_SIZE = 16 + EVAL_TASKS_SLOW = { + "arc_challenge": { + "chat_template": True, + "acc": {"value": 0.2696245733788396, "floor_pct": 0.04}, + "acc_norm": {"value": 0.30716723549488056, "floor_pct": 0.04}, + }, + } + EVAL_TASKS_FAST = ModelTest.derive_fast_eval_tasks(EVAL_TASKS_SLOW) + MODEL_COMPAT_FAST_LAYER_POSITION = "first" + + def test_hunyuan_ocr(self): + self.quantize_and_evaluate() diff --git a/tests/test_hunyuan_ocr_support.py b/tests/test_hunyuan_ocr_support.py new file mode 100644 index 000000000..f07e95d47 --- /dev/null +++ b/tests/test_hunyuan_ocr_support.py @@ -0,0 +1,165 @@ +from types import SimpleNamespace + +import torch +from torch import nn + +from gptqmodel.models import auto +from gptqmodel.models.base import BaseQModel +from gptqmodel.models.definitions import hunyuan_vl as hunyuan_vl_module +from gptqmodel.models.definitions.hunyuan_vl import HunYuanVLQModel +from gptqmodel.utils.hf import normalize_hf_config_compat +from gptqmodel.utils.model import MODALITY +from transformers import AutoConfig + + +def test_hunyuan_vl_model_type_selects_definition(monkeypatch): + fake_config = SimpleNamespace(model_type="hunyuan_vl") + + monkeypatch.setattr( + auto, + "resolve_trust_remote_code", + lambda path, trust_remote_code=False: trust_remote_code, + ) + monkeypatch.setattr(auto, "patch_remote_code_before_config_load", lambda path: None) + monkeypatch.setattr( + auto.AutoConfig, + "from_pretrained", + lambda *args, **kwargs: fake_config, + ) + + assert auto.check_and_get_model_definition("tencent/HunyuanOCR") is HunYuanVLQModel + + +def test_hunyuan_vl_module_tree_matches_dense_text_backbone(): + layer_modules = HunYuanVLQModel.simple_layer_modules( + model_config=SimpleNamespace(), + quantize_config=SimpleNamespace(dynamic=None), + ) + flat_modules = {name for block in layer_modules for name in block} + + assert HunYuanVLQModel.__bases__ == (BaseQModel,) + assert HunYuanVLQModel.modality == [MODALITY.TEXT, MODALITY.IMAGE_TO_TEXT] + assert HunYuanVLQModel.require_load_processor is True + assert HunYuanVLQModel.extract_layers_node() == ["model.language_model.layers"] + assert "self_attn.q_proj" in flat_modules + assert "self_attn.k_proj" in flat_modules + assert "self_attn.v_proj" in flat_modules + assert "self_attn.o_proj" in flat_modules + assert "self_attn.query_layernorm" not in flat_modules + assert "self_attn.key_layernorm" not in flat_modules + assert "mlp.gate_proj" in flat_modules + assert "mlp.up_proj" in flat_modules + assert "mlp.down_proj" in flat_modules + + +def test_hunyuan_vl_keeps_multimodal_and_embedding_modules_in_base_dtype(): + model = nn.Module() + model.model = nn.Module() + model.model.language_model = nn.Module() + model.model.language_model.embed_tokens = nn.Embedding(8, 4) + model.model.language_model.layers = nn.ModuleList([nn.Identity()]) + model.model.language_model.norm = nn.LayerNorm(4) + model.model.language_model.rotary_emb = nn.Identity() + model.model.vision_tower = nn.Linear(4, 4) + + base_modules = set(HunYuanVLQModel.get_base_modules(model)) + + assert base_modules == { + "model.language_model.embed_tokens", + "model.language_model.norm", + "model.language_model.rotary_emb", + "model.vision_tower", + } + + +def test_hunyuan_vl_materialize_uses_canonical_device_and_module_path(): + qmodel = object.__new__(HunYuanVLQModel) + nn.Module.__init__(qmodel) + qmodel.quantize_config = SimpleNamespace(device="cuda") + parent = nn.Module() + parent.embed_tokens = nn.Embedding(8, 4, device="meta") + calls = [] + + def fake_materialize(module, device, *, module_path): + calls.append((module, device, module_path)) + return nn.Embedding(8, 4) + + qmodel.shell_module_materialize = fake_materialize + qmodel._materialize_module( + parent, + "embed_tokens", + "model.language_model.embed_tokens", + ) + + assert parent.embed_tokens.weight.device == torch.device("cpu") + assert calls[0][1:] == ( + torch.device("cuda:0"), + "model.language_model.embed_tokens", + ) + + +def test_hunyuan_vl_prepare_dataset_uses_processor_chat_template(monkeypatch): + calls = [] + + class FakeProcessor: + def apply_chat_template(self, batch, **kwargs): + calls.append((batch, kwargs)) + return {"input_ids": torch.ones((len(batch), 4), dtype=torch.long)} + + monkeypatch.setattr( + hunyuan_vl_module.AutoProcessor, + "from_pretrained", + lambda *args, **kwargs: FakeProcessor(), + ) + + qmodel = object.__new__(HunYuanVLQModel) + nn.Module.__init__(qmodel) + qmodel.model_local_path = "tencent/HunyuanOCR" + samples = [ + [{"role": "user", "content": "first"}], + [{"role": "user", "content": "second"}], + ] + + prepared = qmodel.prepare_dataset(samples, batch_size=2) + + assert len(prepared) == 1 + assert prepared[0]["input_ids"].shape == (2, 4) + assert calls == [ + ( + samples, + { + "add_generation_prompt": True, + "tokenize": True, + "return_dict": True, + "return_tensors": "pt", + "processor_kwargs": {"padding": True}, + }, + ) + ] + + +def test_hunyuan_vl_rope_normalization_stays_in_text_config(tmp_path): + mrope_section = [16, 16, 16, 16] + config = AutoConfig.for_model( + "hunyuan_vl", + text_config={ + "head_dim": 128, + "rope_parameters": { + "alpha": 1000.0, + "factor": 1.0, + "mrope_section": mrope_section, + "rope_theta": 10000.0, + "rope_type": "dynamic", + }, + }, + ) + + normalize_hf_config_compat(config) + + assert "rope_parameters" not in config.to_dict() + assert config.text_config.rope_parameters["mrope_section"] == mrope_section + + config.save_pretrained(tmp_path) + reloaded = AutoConfig.from_pretrained(tmp_path) + + assert reloaded.text_config.rope_parameters["mrope_section"] == mrope_section