Felipe97/llama-cpp-compiled
01.2k
1from __future__ import annotations2 3import json4import math5 6from typing import Callable, Iterable, TYPE_CHECKING7 8import torch9 10if TYPE_CHECKING:11 from torch import Tensor12 13from .base import MmprojModel, ModelBase, SentencePieceTokenTypes, TextModel, gguf, logger14 15 16@ModelBase.register("PhiForCausalLM")17@ModelBase.example("microsoft/phi-2")18class Phi2Model(TextModel):19 model_arch = gguf.MODEL_ARCH.PHI220 21 def set_gguf_parameters(self):22 rot_pct = self.rope_parameters["partial_rotary_factor"]23 n_embd = self.find_hparam(["hidden_size", "n_embd"])24 n_head = self.find_hparam(["num_attention_heads", "n_head"])25 26 self.gguf_writer.add_context_length(self.find_hparam(["n_positions", "max_position_embeddings"]))27 28 self.gguf_writer.add_embedding_length(n_embd)29 self.gguf_writer.add_feed_forward_length(4 * n_embd)30 self.gguf_writer.add_block_count(self.block_count)31 self.gguf_writer.add_head_count(n_head)32 self.gguf_writer.add_head_count_kv(n_head)33 self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_epsilon", "layer_norm_eps"]))34 self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)35 self.gguf_writer.add_file_type(self.ftype)36 self.gguf_writer.add_add_bos_token(False)37 38 39@ModelBase.register("Phi3ForCausalLM", "Phi4ForCausalLMV")40@ModelBase.example("microsoft/Phi-3-mini-4k-instruct")41class Phi3MiniModel(TextModel):42 model_arch = gguf.MODEL_ARCH.PHI343 44 def set_vocab(self):45 # Phi-4 model uses GPT2Tokenizer46 tokenizer_config_file = self.dir_model / 'tokenizer_config.json'47 if tokenizer_config_file.is_file():48 with open(tokenizer_config_file, "r", encoding="utf-8") as f:49 tokenizer_config_json = json.load(f)50 tokenizer_class = tokenizer_config_json['tokenizer_class']51 if tokenizer_class == 'GPT2Tokenizer':52 return self._set_vocab_gpt2()53 54 from sentencepiece import SentencePieceProcessor55 56 tokenizer_path = self.dir_model / 'tokenizer.model'57 58 if not tokenizer_path.is_file():59 raise ValueError(f'Error: Missing {tokenizer_path}')60 61 tokenizer = SentencePieceProcessor()62 tokenizer.LoadFromFile(str(tokenizer_path))63 64 vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())65 66 tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]67 scores: list[float] = [-10000.0] * vocab_size68 toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size69 70 for token_id in range(tokenizer.vocab_size()):71 72 piece = tokenizer.IdToPiece(token_id)73 text = piece.encode("utf-8")74 score = tokenizer.GetScore(token_id)75 76 toktype = SentencePieceTokenTypes.NORMAL77 if tokenizer.IsUnknown(token_id):78 toktype = SentencePieceTokenTypes.UNKNOWN79 elif tokenizer.IsControl(token_id):80 toktype = SentencePieceTokenTypes.CONTROL81 elif tokenizer.IsUnused(token_id):82 toktype = SentencePieceTokenTypes.UNUSED83 elif tokenizer.IsByte(token_id):84 toktype = SentencePieceTokenTypes.BYTE85 86 tokens[token_id] = text87 scores[token_id] = score88 toktypes[token_id] = toktype89 90 added_tokens_file = self.dir_model / 'added_tokens.json'91 if added_tokens_file.is_file():92 with open(added_tokens_file, "r", encoding="utf-8") as f:93 added_tokens_json = json.load(f)94 95 for key in added_tokens_json:96 token_id = added_tokens_json[key]97 if token_id >= vocab_size:98 logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')99 continue100 101 tokens[token_id] = key.encode("utf-8")102 scores[token_id] = -1000.0103 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED104 105 tokenizer_config_file = self.dir_model / 'tokenizer_config.json'106 if tokenizer_config_file.is_file():107 with open(tokenizer_config_file, "r", encoding="utf-8") as f:108 tokenizer_config_json = json.load(f)109 added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})110 for token_id, foken_data in added_tokens_decoder.items():111 token_id = int(token_id)112 token = foken_data["content"].encode("utf-8")113 if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:114 if tokens[token_id] != token:115 logger.warning(f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token.decode("utf-8")!r}')116 tokens[token_id] = token117 scores[token_id] = -1000.0118 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED119 if foken_data.get("special"):120 toktypes[token_id] = SentencePieceTokenTypes.CONTROL121 122 tokenizer_file = self.dir_model / 'tokenizer.json'123 if tokenizer_file.is_file():124 with open(tokenizer_file, "r", encoding="utf-8") as f:125 tokenizer_json = json.load(f)126 added_tokens = tokenizer_json.get("added_tokens", [])127 for foken_data in added_tokens:128 token_id = int(foken_data["id"])129 token = foken_data["content"].encode("utf-8")130 if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:131 if tokens[token_id] != token:132 logger.warning(f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token.decode("utf-8")!r}')133 tokens[token_id] = token134 scores[token_id] = -1000.0135 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED136 if foken_data.get("special"):137 toktypes[token_id] = SentencePieceTokenTypes.CONTROL138 139 self.gguf_writer.add_tokenizer_model("llama")140 self.gguf_writer.add_tokenizer_pre("default")141 self.gguf_writer.add_token_list(tokens)142 self.gguf_writer.add_token_scores(scores)143 self.gguf_writer.add_token_types(toktypes)144 145 special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))146 special_vocab.add_to_gguf(self.gguf_writer)147 148 def set_gguf_parameters(self):149 n_embd = self.find_hparam(["hidden_size", "n_embd"])150 n_head = self.find_hparam(["num_attention_heads", "n_head"])151 n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"])152 rms_eps = self.find_hparam(["rms_norm_eps"])153 max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])154 orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"]155 rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0)156 rope_dims = int(rot_pct * n_embd) // n_head157 158 self.gguf_writer.add_context_length(max_pos_embds)159 self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds)160 self.gguf_writer.add_embedding_length(n_embd)161 self.gguf_writer.add_feed_forward_length(self.find_hparam(["intermediate_size"]))162 self.gguf_writer.add_block_count(self.block_count)163 self.gguf_writer.add_head_count(n_head)164 self.gguf_writer.add_head_count_kv(n_head_kv)165 self.gguf_writer.add_layer_norm_rms_eps(rms_eps)166 self.gguf_writer.add_rope_dimension_count(rope_dims)167 self.gguf_writer.add_rope_freq_base(self.rope_parameters.get("full_attention", self.rope_parameters)["rope_theta"])168 self.gguf_writer.add_file_type(self.ftype)169 sliding_window = self.hparams.get("sliding_window")170 # use zero value of sliding_window to distinguish Phi-4 from other PHI3 models171 if sliding_window is None:172 sliding_window = 0173 self.gguf_writer.add_sliding_window(sliding_window)174 175 def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:176 n_embd = self.find_hparam(["hidden_size", "n_embd"])177 n_head = self.find_hparam(["num_attention_heads", "n_head"])178 max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"])179 orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"]180 rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0)181 rope_dims = int(rot_pct * n_embd) // n_head182 183 # write rope scaling for long context (128k) model184 long_factors = self.rope_parameters.get('long_factor')185 short_factors = self.rope_parameters.get('short_factor')186 if not long_factors:187 return188 189 scale = max_pos_embds / orig_max_pos_embds190 191 rope_scaling_type = self.rope_parameters.get('rope_type', '').lower()192 if len(rope_scaling_type) == 0:193 raise KeyError('Missing the required key rope_scaling.type')194 195 if rope_scaling_type == 'su' or rope_scaling_type == 'longrope':196 attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0197 elif rope_scaling_type == 'yarn':198 attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0199 else:200 raise NotImplementedError(f'The rope scaling type {rope_scaling_type} is not supported yet')201 202 self.gguf_writer.add_rope_scaling_attn_factors(attn_factor)203 204 if long_factors is None or short_factors is None:205 raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor')206 207 if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2:208 raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}. long_factors = {len(long_factors)}, short_factors = {len(short_factors)}.')209 210 yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG), torch.tensor(long_factors, dtype=torch.float32))211 yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT), torch.tensor(short_factors, dtype=torch.float32))212 213 214@ModelBase.register("Phi4ForCausalLMV")215# [TAG_HF_EXAMPLE_MISSING]216class Phi4VisionMmprojModel(MmprojModel):217 def __init__(self, *args, **kwargs):218 super().__init__(*args, **kwargs)219 assert self.hparams_vision is not None220 221 self.vision_total_layers = int(self.find_vparam(self.n_block_keys))222 if self.vision_total_layers < 2:223 raise ValueError(224 f"Phi-4 vision mmproj conversion requires at least 2 vision layers, got {self.vision_total_layers}"225 )226 227 # Phi-4 uses SigLIP2 hidden_states[-2], so export one fewer encoder block and228 # drop post-layernorm/head weights. This makes the GGUF runtime output match229 # the feature map consumed by the patched siglip.cpp Phi-4 projector path.230 self.vision_export_layers = self.vision_total_layers - 1231 self.vision_last_layer_idx = self.vision_total_layers - 1232 233 for key in self.n_block_keys:234 if key in self.hparams_vision:235 self.hparams_vision[key] = self.vision_export_layers236 break237 238 self.block_count = self.vision_export_layers239 self.tensor_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.MMPROJ, self.block_count)240 241 patch_size = self.preprocessor_config.get("patch_size")242 if patch_size is None:243 raise KeyError("Phi-4 vision mmproj conversion requires patch_size in preprocessor_config.json")244 245 self.hparams_vision["patch_size"] = patch_size246 247 pos_emb_name = next(248 (249 name for name in self.model_tensors250 if name.endswith("vision_model.embeddings.position_embedding.weight")251 ),252 None,253 )254 if pos_emb_name is None:255 raise KeyError("Phi-4 vision mmproj conversion could not find position_embedding.weight")256 257 pos_emb_shape = self.model_tensors[pos_emb_name]().shape258 base_grid_tokens = int(pos_emb_shape[0])259 grid_side = math.isqrt(base_grid_tokens)260 if grid_side * grid_side != base_grid_tokens:261 raise ValueError(f"Unexpected Phi-4 position embedding shape: {tuple(pos_emb_shape)}")262 263 self.hparams_vision["image_size"] = grid_side * patch_size264 265 min_num_patches = self.preprocessor_config.get("min_num_patches", self.global_config.get("min_num_patches"))266 max_num_patches = self.preprocessor_config.get("max_num_patches", self.global_config.get("max_num_patches"))267 if min_num_patches is None or max_num_patches is None:268 raise KeyError("Phi-4 vision mmproj conversion requires min_num_patches and max_num_patches")269 270 self.min_pixels = int(min_num_patches) * patch_size * patch_size271 self.max_pixels = int(max_num_patches) * patch_size * patch_size272 273 def set_gguf_parameters(self):274 super().set_gguf_parameters()275 assert self.hparams_vision is not None276 277 self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.PHI4)278 self.gguf_writer.add_vision_min_pixels(self.min_pixels)279 self.gguf_writer.add_vision_max_pixels(self.max_pixels)280 self.gguf_writer.add_vision_use_gelu(True)281 self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("layer_norm_eps", 1e-6))282 283 @classmethod284 def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:285 name, gen = item286 287 name = name.replace("model.vision_tower.vision_tower.", "vision_tower.")288 289 if not name.startswith(("vision_tower.", "model.mm_projector.", "mm_projector.")):290 return None291 292 if ".vision_model.head." in name:293 return None294 295 if ".vision_model.post_layernorm." in name:296 return None297 298 return super().filter_tensors((name, gen))299 300 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:301 if name.startswith("vision_tower."):302 if bid is not None and bid == self.vision_last_layer_idx:303 return304 305 if name.endswith("vision_model.embeddings.patch_embedding.weight"):306 assert self.hparams_vision is not None307 if data_torch.ndim != 2:308 raise ValueError(f"Unexpected Phi-4 patch embedding shape: {tuple(data_torch.shape)}")309 310 patch_area = self.hparams_vision["patch_size"] ** 2311 in_features = data_torch.shape[1]312 if in_features % patch_area != 0:313 raise ValueError(314 f"Phi-4 patch embedding input dim {in_features} is not divisible by patch area {patch_area}"315 )316 317 num_channels = in_features // patch_area318 patch_size = self.hparams_vision["patch_size"]319 data_torch = data_torch.view(data_torch.shape[0], patch_size, patch_size, num_channels)320 data_torch = data_torch.permute(0, 3, 1, 2)321 322 yield from super().modify_tensors(data_torch, name, bid)323 return324 325 if name.startswith(("model.mm_projector.", "mm_projector.")):326 local_name = name327 local_name = local_name.replace("model.mm_projector.", "")328 local_name = local_name.replace("mm_projector.", "")329 330 if not (local_name.startswith("0.") or local_name.startswith("2.")):331 return332 333 suffix = ".bias" if local_name.endswith(".bias") else ".weight"334 mm_idx = int(local_name.split(".", maxsplit=1)[0])335 yield (self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, mm_idx, suffix=suffix), data_torch)336 return337 338 return339 340 341@ModelBase.register("PhiMoEForCausalLM")342@ModelBase.example("microsoft/Phi-3.5-MoE-instruct")343class PhiMoeModel(Phi3MiniModel):344 model_arch = gguf.MODEL_ARCH.PHIMOE345 346 _experts: list[dict[str, Tensor]] | None = None347 348 def set_gguf_parameters(self):349 super().set_gguf_parameters()350 self.gguf_writer.add_expert_used_count(self.find_hparam(["num_experts_per_tok", "num_experts_per_token"]))351 self.gguf_writer.add_expert_count(self.find_hparam(["num_local_experts", "num_experts"]))352 353 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:354 # process the experts separately355 if name.find("block_sparse_moe.experts") != -1:356 n_experts = self.find_hparam(["num_local_experts", "num_experts"])357 assert bid is not None358 359 if self._experts is None:360 self._experts = [{} for _ in range(self.block_count)]361 362 self._experts[bid][name] = data_torch363 364 if len(self._experts[bid]) >= n_experts * 3:365 # merge the experts into a single 3d tensor366 for w_name in ["w1", "w2", "w3"]:367 datas: list[Tensor] = []368 369 for xid in range(n_experts):370 ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{w_name}.weight"371 datas.append(self._experts[bid][ename])372 del self._experts[bid][ename]373 374 data_torch = torch.stack(datas, dim=0)375 376 merged_name = f"model.layers.{bid}.block_sparse_moe.experts.{w_name}.weight"377 378 yield from super().modify_tensors(data_torch, merged_name, bid)379 return380 else:381 return382 383 yield from super().modify_tensors(data_torch, name, bid)384 385 def prepare_tensors(self):386 super().prepare_tensors()387 388 if self._experts is not None:389 # flatten `list[dict[str, Tensor]]` into `list[str]`390 experts = [k for d in self._experts for k in d.keys()]391 if len(experts) > 0:392 raise ValueError(f"Unprocessed experts: {experts}")393