Brunobkr/llama.cpp_AlgMor24_github

1from __future__ import annotations2 3import json4import os5 6from pathlib import Path7from typing import Any, Callable, Iterable, TYPE_CHECKING8 9import torch10 11if TYPE_CHECKING:12    from torch import Tensor13 14from .base import ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger15 16 17@ModelBase.register("BertModel", "BertForMaskedLM", "CamembertModel", "BertForSequenceClassification")18class BertModel(TextModel):19    model_arch = gguf.MODEL_ARCH.BERT20 21    def __init__(self, *args, **kwargs):22        super().__init__(*args, **kwargs)23        self.vocab_size = None24 25        if cls_out_labels := self.hparams.get("id2label"):26            if len(cls_out_labels) == 2 and cls_out_labels[0] == "LABEL_0":27                # Remove dummy labels added by AutoConfig28                cls_out_labels = None29        self.cls_out_labels = cls_out_labels30 31    def set_gguf_parameters(self):32        super().set_gguf_parameters()33        self.gguf_writer.add_causal_attention(False)34        self._try_set_pooling_type()35 36        if self.cls_out_labels:37            self.gguf_writer.add_classifier_output_labels([v for k, v in sorted(self.cls_out_labels.items())])38 39    def set_vocab(self):40        tokens, toktypes, tokpre = self.get_vocab_base()41        self.vocab_size = len(tokens)42 43        # we need this to validate the size of the token_type embeddings44        # though currently we are passing all zeros to the token_type embeddings45        # "Sequence A" or "Sequence B"46        self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))47 48        # convert to phantom space vocab49        def phantom(tok, toktype):50            if toktype == gguf.TokenType.CONTROL:51                return tok52            if tok.startswith("##"):53                return tok[2:]54            return "\u2581" + tok55        assert len(tokens) == len(toktypes)56        tokens = list(map(phantom, tokens, toktypes))57 58        # add vocab to gguf59        self.gguf_writer.add_tokenizer_model("bert")60        self.gguf_writer.add_tokenizer_pre(tokpre)61        self.gguf_writer.add_token_list(tokens)62        self.gguf_writer.add_token_types(toktypes)63 64        # handle special tokens65        special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))66        special_vocab.add_to_gguf(self.gguf_writer)67 68    @classmethod69    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:70        name, gen = item71 72        if name.startswith("bert."):73            name = name[5:]74 75        if name.endswith(".gamma"):76            name = name[:-6] + ".weight"77 78        if name.endswith(".beta"):79            name = name[:-5] + ".bias"80 81        # we are only using BERT for embeddings so we don't need the pooling layer82        if name in ("embeddings.position_ids", "pooler.dense.weight", "pooler.dense.bias"):83            return None84 85        if name.startswith("cls.predictions"):86            return None87 88        if name.startswith("cls.seq_relationship"):89            return None90 91        return super().filter_tensors((name, gen))92 93    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:94        if self.cls_out_labels:95            # For BertForSequenceClassification (direct projection layer)96            if name == "classifier.weight":97                name = "classifier.out_proj.weight"98 99            if name == "classifier.bias":100                name = "classifier.out_proj.bias"101 102        yield from super().modify_tensors(data_torch, name, bid)103 104    def _xlmroberta_tokenizer_init(self) -> None:105        # we need the pad_token_id to know how to chop down position_embd matrix106        if (pad_token_id := self.hparams.get("pad_token_id")) is not None:107            self._position_offset = 1 + pad_token_id108            if "max_position_embeddings" in self.hparams:109                self.hparams["max_position_embeddings"] -= self._position_offset110        else:111            self._position_offset = None112 113    def _xlmroberta_set_vocab(self) -> None:114        # to avoid TypeError: Descriptors cannot be created directly115        # exception when importing sentencepiece_model_pb2116        os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"117        from sentencepiece import SentencePieceProcessor118        from sentencepiece import sentencepiece_model_pb2 as model119 120        tokenizer_path = self.dir_model / 'sentencepiece.bpe.model'121 122        tokenizer_json = {}123        tokenizer_config_json = {}124        if not tokenizer_path.is_file():125            tokenizer_path = self.dir_model / 'tokenizer.json'126            tokenizer_config_path = self.dir_model / 'tokenizer_config.json'127 128            if not tokenizer_path.is_file():129                raise FileNotFoundError(f"File not found: {tokenizer_path}")130 131            from base64 import b64decode132            from transformers import AutoTokenizer133            tokenizer = AutoTokenizer.from_pretrained(self.dir_model)134 135            with open(tokenizer_path, "r", encoding="utf-8") as fp:136                tokenizer_json = json.load(fp)137 138            if tokenizer_config_path.is_file():139                with open(tokenizer_config_path, "r", encoding="utf-8") as fp:140                    tokenizer_config_json = json.load(fp)141 142            add_prefix = tokenizer.add_prefix_space  # ty: ignore[unresolved-attribute]143            remove_whitespaces = tokenizer.clean_up_tokenization_spaces  # ty: ignore[unresolved-attribute]144            precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])145 146            vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size)  # ty: ignore[unresolved-attribute]147        else:148            sentencepiece_model = model.ModelProto()  # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute]149            sentencepiece_model.ParseFromString(open(tokenizer_path, "rb").read())150            assert sentencepiece_model.trainer_spec.model_type == 1  # UNIGRAM151 152            add_prefix = sentencepiece_model.normalizer_spec.add_dummy_prefix153            remove_whitespaces = sentencepiece_model.normalizer_spec.remove_extra_whitespaces154            precompiled_charsmap = sentencepiece_model.normalizer_spec.precompiled_charsmap155 156            tokenizer = SentencePieceProcessor()157            tokenizer.LoadFromFile(str(tokenizer_path))158 159            vocab_size = max(self.hparams.get("vocab_size", 0), tokenizer.vocab_size())160 161        tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]162        scores: list[float] = [-10000.0] * vocab_size163        toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size164 165        if isinstance(tokenizer, SentencePieceProcessor):166            for token_id in range(tokenizer.vocab_size()):167                piece = tokenizer.IdToPiece(token_id)168                text = piece.encode("utf-8")169                score = tokenizer.GetScore(token_id)170 171                toktype = SentencePieceTokenTypes.NORMAL172                if tokenizer.IsUnknown(token_id):173                    toktype = SentencePieceTokenTypes.UNKNOWN174                elif tokenizer.IsControl(token_id):175                    toktype = SentencePieceTokenTypes.CONTROL176                elif tokenizer.IsUnused(token_id):177                    toktype = SentencePieceTokenTypes.UNUSED178                elif tokenizer.IsByte(token_id):179                    toktype = SentencePieceTokenTypes.BYTE180 181                tokens[token_id] = text182                scores[token_id] = score183                toktypes[token_id] = toktype184        else:185            added_vocab = tokenizer.get_added_vocab()  # ty: ignore[unresolved-attribute]186            unk_token = tokenizer_config_json.get("unk_token")187            unk_token_id = added_vocab.get(unk_token, tokenizer_json["model"].get("unk_id", 3))  # ty: ignore[no-matching-overload]188 189            for token_id in range(tokenizer.vocab_size):  # ty: ignore[unresolved-attribute]190                piece = tokenizer._convert_id_to_token(token_id)  # ty: ignore[unresolved-attribute]191                if (piece := tokenizer._convert_id_to_token(token_id)) is not None:  # ty: ignore[unresolved-attribute]192                    text = piece.encode("utf-8")193                    score = tokenizer_json["model"]["vocab"][token_id][1]194 195                    toktype = SentencePieceTokenTypes.NORMAL196                    if token_id == unk_token_id:197                        toktype = SentencePieceTokenTypes.UNKNOWN198                    elif token_id in tokenizer.all_special_ids:  # ty: ignore[unresolved-attribute]199                        toktype = SentencePieceTokenTypes.CONTROL200                    elif token_id in added_vocab.values():201                        toktype = SentencePieceTokenTypes.USER_DEFINED202                    # No reliable way to detect this, but jina doesn't have any203                    # elif tokenizer.IsByte(token_id):204                    #     toktype = SentencePieceTokenTypes.BYTE205 206                    tokens[token_id] = text207                    scores[token_id] = score208                    toktypes[token_id] = toktype209 210        if isinstance(tokenizer, SentencePieceProcessor):211            # realign tokens (see HF tokenizer code)212            tokens = [b'<s>', b'<pad>', b'</s>', b'<unk>'] + tokens[3:-1]213            scores = [0.0, 0.0, 0.0, 0.0] + scores[3:-1]214            toktypes = [215                SentencePieceTokenTypes.CONTROL,216                SentencePieceTokenTypes.CONTROL,217                SentencePieceTokenTypes.CONTROL,218                SentencePieceTokenTypes.UNKNOWN,219            ] + toktypes[3:-1]220 221            if self.model_arch == gguf.MODEL_ARCH.NOMIC_BERT_MOE:222                # Add mask token missing from sentencepiece.bpe.model223                tokens[250001] = b'<mask>'224                scores[250001] = 0.0225                toktypes[250001] = SentencePieceTokenTypes.CONTROL226 227        self.gguf_writer.add_tokenizer_model("t5")228        self.gguf_writer.add_tokenizer_pre("default")229        self.gguf_writer.add_token_list(tokens)230        self.gguf_writer.add_token_scores(scores)231        self.gguf_writer.add_token_types(toktypes)232        self.gguf_writer.add_add_space_prefix(add_prefix)233        self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))234        self.gguf_writer.add_remove_extra_whitespaces(remove_whitespaces)235        if precompiled_charsmap:236            self.gguf_writer.add_precompiled_charsmap(precompiled_charsmap)237 238        special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))239        special_vocab.add_to_gguf(self.gguf_writer)240 241 242@ModelBase.register("DistilBertModel", "DistilBertForMaskedLM", "DistilBertForSequenceClassification")243class DistilBertModel(BertModel):244    model_arch = gguf.MODEL_ARCH.BERT245 246    def set_gguf_parameters(self):247        self.gguf_writer.add_layer_norm_eps(1e-12)248        logger.info("gguf: layer norm epsilon = 1e-12")249        super().set_gguf_parameters()250 251    @classmethod252    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:253        name, gen = item254 255        if name.startswith("distilbert."):256            name = name[11:]257 258        # These layers act as MLM head, so we don't need them259        if name.startswith("vocab_"):260            return None261 262        return super().filter_tensors((name, gen))263 264 265@ModelBase.register("RobertaModel", "RobertaForSequenceClassification")266class RobertaModel(BertModel):267    model_arch = gguf.MODEL_ARCH.BERT268 269    def __init__(self, *args, **kwargs):270        super().__init__(*args, **kwargs)271 272        # we need the pad_token_id to know how to chop down position_embd matrix273        if (pad_token_id := self.hparams.get("pad_token_id")) is not None:274            self._position_offset = 1 + pad_token_id275            if "max_position_embeddings" in self.hparams:276                self.hparams["max_position_embeddings"] -= self._position_offset277        else:278            self._position_offset = None279 280    def set_vocab(self):281        """Support BPE tokenizers for roberta models"""282        bpe_tok_path = self.dir_model / "tokenizer.json"283        if bpe_tok_path.exists():284            self._set_vocab_gpt2()285 286            # we need this to validate the size of the token_type embeddings287            # though currently we are passing all zeros to the token_type embeddings288            # "Sequence A" or "Sequence B"289            self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))290 291        else:292            return super().set_vocab()293 294    @classmethod295    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:296        name, gen = item297 298        # if name starts with "roberta.", remove the prefix299        # e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main300        if name.startswith("roberta."):301            name = name[8:]302 303        return super().filter_tensors((name, gen))304 305    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:306        # position embeddings start at pad_token_id + 1, so just chop down the weight tensor307        if name == "embeddings.position_embeddings.weight":308            if self._position_offset is not None:309                data_torch = data_torch[self._position_offset:,:]310 311        yield from super().modify_tensors(data_torch, name, bid)312 313 314@ModelBase.register("NomicBertModel")315class NomicBertModel(BertModel):316    model_arch = gguf.MODEL_ARCH.BERT317 318    def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, **kwargs: Any):319        hparams = kwargs.pop("hparams", None)320        if hparams is None:321            hparams = ModelBase.load_hparams(dir_model, False)322 323        self.is_moe = bool(hparams.get("moe_every_n_layers"))324        self.model_arch = gguf.MODEL_ARCH.NOMIC_BERT_MOE if self.is_moe else gguf.MODEL_ARCH.NOMIC_BERT325 326        super().__init__(dir_model, ftype, fname_out, hparams=hparams, **kwargs)327 328        self._tokenizer_is_xlmroberta = self._is_tokenizer_xlmroberta()329        if self._tokenizer_is_xlmroberta:330            self._xlmroberta_tokenizer_init()331 332        npos, mtp = self.hparams["n_positions"], self.hparams.get("max_trained_positions", 2048)333        if npos == 8192 and mtp == 2048:334            self.hparams["n_positions"] = 2048  # nomic-embed-text v1 and v1.5 are trained for 2048 tokens.335        elif npos == 2048 and mtp == 2048:336            self.hparams["n_positions"] = 512   # nomic-embed-text-v2-moe is trained for 512 tokens.337        else:338            raise ValueError(f"unrecognized parameters: n_positions={npos}, max_trained_positions={mtp}")339 340        assert self.hparams["activation_function"] == "gelu" if self.is_moe else "swiglu"341 342        # this doesn't do anything in the HF version343        assert self.hparams["causal"] is False344        # no bias tensors unless MoE345        assert self.hparams["qkv_proj_bias"] == self.is_moe346        assert self.hparams["mlp_fc1_bias"]  == self.is_moe347        assert self.hparams["mlp_fc2_bias"]  == self.is_moe348 349        # norm at end of layer350        assert self.hparams["prenorm"] is False351        # standard RoPE352        assert self.hparams["rotary_emb_fraction"] == 1.0353        assert self.hparams["rotary_emb_interleaved"] is False354        assert self.hparams["rotary_emb_scale_base"] is None355 356    def set_vocab(self) -> None:357        if self._tokenizer_is_xlmroberta:358            return self._xlmroberta_set_vocab()359        return super().set_vocab()360 361    @classmethod362    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:363        name, gen = item364 365        # If the tensor is an experts bias tensor, skip it.366        if "mlp.experts.bias" in name:367            return None368 369        return super().filter_tensors(item)370 371    def modify_tensors(self, data_torch: torch.Tensor, name: str, bid: int | None) -> Iterable[tuple[str, torch.Tensor]]:372        if "mlp.experts.mlp.w1" in name:373            n_experts = self.find_hparam(["num_local_experts", "num_experts"])374            data_torch = data_torch.view(n_experts, self.hparams["n_inner"], self.hparams["n_embd"])375            name += ".weight"376 377        if "mlp.experts.mlp.w2" in name:378            n_experts = self.find_hparam(["num_local_experts", "num_experts"])379            data_torch = data_torch.view(n_experts, self.hparams["n_inner"], self.hparams["n_embd"])380            data_torch = data_torch.transpose(1, 2)381            name += ".weight"382 383        yield from super().modify_tensors(data_torch, name, bid)384 385    def set_gguf_parameters(self):386        super().set_gguf_parameters()387        if self.is_moe:388            self.gguf_writer.add_moe_every_n_layers(self.hparams["moe_every_n_layers"])389            self.gguf_writer.add_expert_used_count(self.hparams["moe_top_k"])390 391    def _is_tokenizer_xlmroberta(self) -> bool:392        with open(self.dir_model / "tokenizer.json") as f:393            tokenizer_json = json.load(f)394        toktyp = tokenizer_json["model"]["type"]395        if toktyp == "Unigram":396            return True397        if toktyp == "WordPiece":398            return False399        raise ValueError(f"unknown tokenizer: {toktyp}")400 401 402@ModelBase.register("NeoBERT", "NeoBERTLMHead", "NeoBERTForSequenceClassification")403class NeoBert(BertModel):404    model_arch = gguf.MODEL_ARCH.NEO_BERT405 406    def set_gguf_parameters(self):407        super().set_gguf_parameters()408 409        # NeoBERT uses 2/3 of the intermediate size as feed forward length410        self.gguf_writer.add_feed_forward_length(int(2 * self.hparams["intermediate_size"] / 3))411        self.gguf_writer.add_rope_freq_base(10000.0)  # default value for NeoBERT412        self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)413 414        f_rms_eps = self.hparams.get("norm_eps", 1e-6)  # default value for NeoBERT415        self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)416        logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")417 418        self.gguf_writer.add_pooling_type(gguf.PoolingType.CLS) # https://huggingface.co/chandar-lab/NeoBERT#how-to-use419 420    @classmethod421    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:422        name, gen = item423 424        if name.startswith("decoder."):425            return None426 427        if name.startswith("model."):428            name = name[6:]429 430        return super().filter_tensors((name, gen))431 432 433@ModelBase.register("EuroBertModel", "JinaEmbeddingsV5Model")434class EuroBertModel(TextModel):435    model_arch = gguf.MODEL_ARCH.EUROBERT436 437    def set_vocab(self):438        self.gguf_writer.add_add_bos_token(False)439        self._set_vocab_gpt2()440 441    def set_gguf_parameters(self):442        super().set_gguf_parameters()443 444        # EuroBert is bidirectional (encoder)445        self.gguf_writer.add_causal_attention(False)446 447        self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)448 449        self._try_set_pooling_type()450 451    @classmethod452    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:453        name, gen = item454 455        if name.startswith("model."):456            name = name[6:]457 458        return super().filter_tensors((name, gen))459 460 461@ModelBase.register("XLMRobertaModel", "XLMRobertaForSequenceClassification")462class XLMRobertaModel(BertModel):463    model_arch = gguf.MODEL_ARCH.BERT464    _lora_files = {}465    _lora_names = []466 467    def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, **kwargs: Any):468        hparams = kwargs.pop("hparams", None)469        if hparams is None:470            hparams = ModelBase.load_hparams(dir_model, False)471 472        if lora_names := hparams.get("lora_adaptations"):473            self._lora_names = lora_names474            self.model_arch = gguf.MODEL_ARCH.JINA_BERT_V3475 476        super().__init__(dir_model, ftype, fname_out, hparams=hparams, **kwargs)477        self._xlmroberta_tokenizer_init()478 479    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:480        if self._lora_names:481            for name in self._lora_names:482                fname = self.add_prefix_to_filename(self.fname_out, f"lora-{name}-")483                self._lora_files[name] = gguf.GGUFWriter(fname, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file, dry_run=self.dry_run)484 485        return super().generate_extra_tensors()486 487    def set_type(self):488        for lora_writer in self._lora_files.values():489            lora_writer.add_type(gguf.GGUFType.ADAPTER)490            lora_writer.add_string(gguf.Keys.Adapter.TYPE, "lora")491        super().set_type()492 493    def set_vocab(self):494        self._xlmroberta_set_vocab()495 496    @classmethod497    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:498        name, gen = item499 500        # if name starts with "roberta.", remove the prefix501        # e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main502        if name.startswith("roberta."):503            name = name[8:]504 505        # jina-embeddings-v3506        if ".parametrizations." in name:507            name = name.replace(".parametrizations.", ".")508            if name.endswith(".original"):509                name = name[:-9]510 511        return super().filter_tensors((name, gen))512 513    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:514        # position embeddings start at pad_token_id + 1, so just chop down the weight tensor515        if name == "embeddings.position_embeddings.weight":516            if self._position_offset is not None:517                data_torch = data_torch[self._position_offset:,:]518 519        if name.endswith(".0.lora_A") or name.endswith(".0.lora_B"):520            if name.startswith("pooler.dense"):521                return522 523            num_loras = data_torch.size(0)524            assert num_loras == len(self._lora_names)525 526            # Split out each LoRA in their own GGUF527            for i, lora_writer in enumerate(self._lora_files.values()):528                new_name = self.map_tensor_name(name[:-9]) + name[-7:].lower()529                data = data_torch[i, :, :]530                # Transpose/flip token_embd/types into correct shape531                if new_name == "token_embd.weight.lora_b":532                    data = data.T533                elif new_name.startswith("token_types.weight."):534                    new_name = new_name[:-1] + ("a" if new_name[-1:] == "b" else "b")535                lora_writer.add_tensor(new_name, data.float().numpy(), raw_dtype=gguf.GGMLQuantizationType.F32)536 537            return538 539        yield from super().modify_tensors(data_torch, name, bid)540 541    def set_gguf_parameters(self):542        super().set_gguf_parameters()543 544        # jina-embeddings-v3545        lora_alpha = self.hparams.get("lora_alpha")546        if lora_prompt_prefixes := self.hparams.get("task_instructions"):547            assert self._lora_files and all(lora_name in lora_prompt_prefixes for lora_name in self._lora_files.keys())548        for lora_name, lora_writer in self._lora_files.items():549            lora_writer.add_float32(gguf.Keys.Adapter.LORA_ALPHA, lora_alpha if lora_alpha is not None else 1.0)550            lora_writer.add_string(gguf.Keys.Adapter.LORA_TASK_NAME, lora_name)551            if lora_prompt_prefixes:552                lora_writer.add_string(gguf.Keys.Adapter.LORA_PROMPT_PREFIX, lora_prompt_prefixes[lora_name])553 554    def write(self):555        super().write()556        for lora_writer in self._lora_files.values():557            lora_writer.write_header_to_file()558            lora_writer.write_kv_data_to_file()559            lora_writer.write_tensors_to_file(progress=True)560            lora_writer.close()561 562 563@ModelBase.register("JinaBertModel", "JinaBertForMaskedLM")564class JinaBertV2Model(BertModel):565    model_arch = gguf.MODEL_ARCH.JINA_BERT_V2566 567    def set_vocab(self):568        tokenizer_class = 'BertTokenizer'569        with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f:570            tokenizer_class = json.load(f)['tokenizer_class']571 572        if tokenizer_class == 'BertTokenizer':573            super().set_vocab()574        elif tokenizer_class == 'RobertaTokenizer':575            pre_tokenizer_type = None576            tokenizer_json_path = self.dir_model / "tokenizer.json"577            if tokenizer_json_path.is_file():578                with open(tokenizer_json_path, "r", encoding="utf-8") as f:579                    pre_tokenizer_type = json.load(f).get("pre_tokenizer", {}).get("type")580 581            if pre_tokenizer_type == "Whitespace":582                self._set_vocab_whitespace()583            else:584                self._set_vocab_gpt2()585            self.gguf_writer.add_token_type_count(2)586        else:587            raise NotImplementedError(f'Tokenizer {tokenizer_class} is not supported for JinaBertModel')588 589 590@ModelBase.register("ModernBertModel", "ModernBertForMaskedLM", "ModernBertForSequenceClassification")591class ModernBertModel(BertModel):592    model_arch = gguf.MODEL_ARCH.MODERN_BERT593 594    def set_vocab(self):595        self.gguf_writer.add_add_bos_token(True)596        self.gguf_writer.add_add_eos_token(True)597        self.gguf_writer.add_add_sep_token(True)598        self._set_vocab_gpt2()599 600    def set_gguf_parameters(self):601        super().set_gguf_parameters()602        self.gguf_writer.add_sliding_window(self.hparams["local_attention"])603        if (sliding_window_pattern := self.hparams.get("global_attn_every_n_layers")) is not None:604            self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)605        self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)606        self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])607        # FFN activation: ModernBert uses a GLU pair (ffn_up output is 2*n_ff). The608        # original ModernBERT uses GELU (-> GeGLU); some derivatives such as IBM609        # Granite Embedding 97m R2 use SiLU (-> SwiGLU). Persist this so the610        # llama.cpp graph can pick the matching activation.611        if hidden_act := self.hparams.get("hidden_activation"):612            self.gguf_writer.add_hidden_act(hidden_act)613 614    @classmethod615    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:616        name, gen = item617 618        if name.startswith("model."):619            name = name[6:]620 621        return super().filter_tensors((name, gen))622 623    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:624        if self.cls_out_labels:625            # For BertForSequenceClassification (direct projection layer)626            if name == "classifier.weight":627                name = "classifier.out_proj.weight"628 629            if name == "classifier.bias":630                name = "classifier.out_proj.bias"631 632        yield from super().modify_tensors(data_torch, name, bid)633