Brunobkr/llama.cpp_AlgMor24_github

1from __future__ import annotations2 3from typing import Callable, TYPE_CHECKING4 5if TYPE_CHECKING:6    from torch import Tensor7 8from .base import MmprojModel, ModelBase, gguf9 10 11@ModelBase.register("Idefics3ForConditionalGeneration", "SmolVLMForConditionalGeneration")12class SmolVLMModel(MmprojModel):13    def __init__(self, *args, **kwargs):14        super().__init__(*args, **kwargs)15        if self.hparams["model_type"] == "smolvlm_vision":16            # fix for SmolVLM2, missing some keys in config.json17            # default values are taken from transformers code18            self.hparams["hidden_size"] = self.hparams.get("hidden_size", 1152)19            self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 16)20            self.hparams["intermediate_size"] = self.hparams.get("intermediate_size", 3072)21 22    def set_gguf_parameters(self):23        super().set_gguf_parameters()24        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.IDEFICS3)25        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams.get("layer_norm_eps", 1e-5))26        self.gguf_writer.add_vision_projector_scale_factor(self.global_config.get("scale_factor", 2))27        self.gguf_writer.add_vision_use_gelu(True)28 29        # Add the preprocessor longest edge size30        preproc_image_size = self.preprocessor_config.get("size", {}).get("longest_edge", self.image_size)31        self.gguf_writer.add_vision_preproc_image_size(preproc_image_size)32 33    def tensor_force_quant(self, name, new_name, bid, n_dims):34        if ".embeddings." in name:35            return gguf.GGMLQuantizationType.F3236        return super().tensor_force_quant(name, new_name, bid, n_dims)37 38    @classmethod39    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:40        name, gen = item41 42        is_vision_tensor = "vision_tower" in name or "vision_model" in name or "model.connector" in name43 44        if not is_vision_tensor:45            return None46 47        return super().filter_tensors(item)48