Brunobkr/llama.cpp_AlgMor24_github

1from __future__ import annotations2 3import math4 5from typing import Any, Iterable, TYPE_CHECKING6 7import torch8 9if TYPE_CHECKING:10    from torch import Tensor11 12from .base import ModelBase, TextModel, gguf13 14 15@ModelBase.register("DeciLMForCausalLM")16class DeciModel(TextModel):17    model_arch = gguf.MODEL_ARCH.DECI18 19    @staticmethod20    def _ffn_mult_to_intermediate_size(ffn_mult: float, n_embd: int) -> int:21        # DeciLM-specific code22        intermediate_size = int(2 * ffn_mult * n_embd / 3)23        return DeciModel._find_multiple(intermediate_size, 256)24 25    @staticmethod26    def _find_multiple(n: int, k: int) -> int:27        # DeciLM-specific code28        if n % k == 0:29            return n30        return n + k - (n % k)31 32    def __init__(self, *args, **kwargs):33        super().__init__(*args, **kwargs)34 35        if "block_configs" in self.hparams: # Llama-3_1-Nemotron-51B36            _block_configs: list[dict[str,Any]] = self.hparams["block_configs"]37            assert self.block_count == len(_block_configs)38            self._num_kv_heads = list()39            self._num_heads = list()40            _ffn_multipliers = list()41            # ***linear attention layer***42            # if n_heads_in_group is None and replace_with_linear is True43            # then _num_kv_heads[il] is 0 and _num_heads[il] is num_attention_heads44            # ***attention-free layer***45            # if n_heads_in_group is None and replace_with_linear is False46            # then _num_kv_heads[il] is 0 and _num_heads[il] is 047            # ***normal attention-layer***48            # if n_heads_in_group is not None, then49            # _num_kv_heads[il] is num_attention_head // n_heads_in_group and50            # _num_heads[il] is num_attention_head51            # ***dummy layer*** for nemotron 253B52            # if n_heads_in_group is None and ffn_mult is None53            # then _num_kv_heads[il] is 0 and _num_heads[il] is 0 and _ffn_dims is 054            for il in range(len(_block_configs)):55                if _block_configs[il]["attention"]["n_heads_in_group"] is None:56                    if _block_configs[il]["attention"]["replace_with_linear"] is True:57                        self._num_kv_heads.append(0)58                        self._num_heads.append(self.hparams["num_attention_heads"])59                    else:60                        self._num_kv_heads.append(0)61                        self._num_heads.append(0)62                else:63                    self._num_kv_heads.append(self.hparams["num_attention_heads"] // _block_configs[il]["attention"]["n_heads_in_group"])64                    self._num_heads.append(self.hparams["num_attention_heads"])65                if _block_configs[il]["ffn"]["ffn_mult"] is None: # dummy layer66                    _ffn_multipliers.append(0.0)67                else:68                    _ffn_multipliers.append(_block_configs[il]["ffn"]["ffn_mult"])69            assert self.block_count == len(self._num_kv_heads)70            assert self.block_count == len(self._num_heads)71            assert self.block_count == len(_ffn_multipliers)72            assert isinstance(self._num_kv_heads, list) and isinstance(self._num_kv_heads[0], int)73            assert isinstance(self._num_heads, list) and isinstance(self._num_heads[0], int)74            assert isinstance(_ffn_multipliers, list) and isinstance(_ffn_multipliers[0], float)75            self._ffn_dims: list[int] = [76                DeciModel._ffn_mult_to_intermediate_size(multiplier, self.hparams["hidden_size"])77                for multiplier in _ffn_multipliers78            ]79 80    def set_vocab(self):81        # Please change tokenizer_config.json of Llama-3_1-Nemotron-51B's82        # eos_token from '|eot_id|' to '|end_of_text|'83        if self.hparams.get("vocab_size", 128256) == 128256:84            tokens, toktypes, tokpre = self.get_vocab_base()85            self.gguf_writer.add_tokenizer_model("gpt2")86            self.gguf_writer.add_tokenizer_pre(tokpre)87            self.gguf_writer.add_token_list(tokens)88            self.gguf_writer.add_token_types(toktypes)89 90            special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)91            special_vocab.add_to_gguf(self.gguf_writer)92        else:93            # DeciLM-7B94            self._set_vocab_llama_hf()95 96    def set_gguf_parameters(self):97        if "block_configs" in self.hparams: # Llama-3_1-Nemotron-51B98            assert self.block_count == len(self._num_kv_heads)99            assert self.block_count == len(self._num_heads)100            assert self.block_count == len(self._ffn_dims)101            if (rope_theta := self.rope_parameters.get("rope_theta")) is not None:102                self.gguf_writer.add_rope_freq_base(rope_theta)103            self.gguf_writer.add_head_count_kv(self._num_kv_heads)104            self.gguf_writer.add_head_count(self._num_heads)105            self.gguf_writer.add_feed_forward_length(self._ffn_dims)106            self.gguf_writer.add_block_count(self.block_count)107            self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])108            self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])109            self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])110            self.gguf_writer.add_key_length(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])111            self.gguf_writer.add_value_length(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])112            self.gguf_writer.add_file_type(self.ftype)113        else: # DeciLM-7B114            super().set_gguf_parameters()115            if "num_key_value_heads_per_layer" in self.hparams: # DeciLM-7B116                self._num_kv_heads: list[int] = self.hparams["num_key_value_heads_per_layer"]117                assert self.block_count == len(self._num_kv_heads)118                self.gguf_writer.add_head_count_kv(self._num_kv_heads)119        hparams = self.hparams120        self.gguf_writer.add_vocab_size(hparams["vocab_size"])121 122        if (rope_dim := hparams.get("head_dim")) is None:123            rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"]124        self.gguf_writer.add_rope_dimension_count(rope_dim)125 126    @staticmethod127    def permute(weights: Tensor, n_head: int, n_head_kv: int | None):128        if n_head_kv is not None and n_head != n_head_kv:129            n_head = n_head_kv130        return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])131                .swapaxes(1, 2)132                .reshape(weights.shape))133 134    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:135        n_head = self.hparams["num_attention_heads"]136        if bid is not None:137            if "num_key_value_heads_per_layer" in self.hparams:138                n_kv_head = self.hparams["num_key_value_heads_per_layer"][bid]139            elif "block_configs" in self.hparams:140                n_kv_head = self._num_kv_heads[bid]141                n_head = self._num_heads[bid]142            else:143                n_kv_head = self.hparams.get("num_key_value_heads")144        else:145            n_kv_head = self.hparams.get("num_key_value_heads")146 147        if name.endswith(("q_proj.weight", "q_proj.bias")):148            data_torch = DeciModel.permute(data_torch, n_head, n_head)149        if name.endswith(("k_proj.weight", "k_proj.bias")):150            data_torch = DeciModel.permute(data_torch, n_head, n_kv_head)151        yield from super().modify_tensors(data_torch, name, bid)152 153    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:154        if rope_params := self.rope_parameters.get("full_attention", self.rope_parameters):155            if rope_params.get("rope_type", '').lower() == "llama3":156                base = rope_params.get("rope_theta", 10000.0)157                if (dim := self.hparams.get("head_dim")) is None:158                    dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]159                freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))160 161                factor = rope_params.get("factor", 8.0)162                low_freq_factor = rope_params.get("low_freq_factor", 1.0)163                high_freq_factor = rope_params.get("high_freq_factor", 4.0)164                old_context_len = rope_params.get("original_max_position_embeddings", 8192)165 166                low_freq_wavelen = old_context_len / low_freq_factor167                high_freq_wavelen = old_context_len / high_freq_factor168                assert low_freq_wavelen != high_freq_wavelen169 170                rope_factors = []171                for freq in freqs:172                    wavelen = 2 * math.pi / freq173                    if wavelen < high_freq_wavelen:174                        rope_factors.append(1)175                    elif wavelen > low_freq_wavelen:176                        rope_factors.append(factor)177                    else:178                        smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)179                        rope_factors.append(1 / ((1 - smooth) / factor + smooth))180 181                yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))182 183    def prepare_tensors(self):184        super().prepare_tensors()185