KBaba7/llama.cpp
0
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3 4from __future__ import annotations5 6import ast7import logging8import argparse9import contextlib10import json11import os12import re13import sys14from enum import IntEnum15from pathlib import Path16from hashlib import sha25617from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, cast18from itertools import chain19 20import math21import numpy as np22import torch23 24if TYPE_CHECKING:25 from torch import Tensor26 27if 'NO_LOCAL_GGUF' not in os.environ:28 sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))29import gguf30 31logger = logging.getLogger("hf-to-gguf")32 33 34###### MODEL DEFINITIONS ######35 36class SentencePieceTokenTypes(IntEnum):37 NORMAL = 138 UNKNOWN = 239 CONTROL = 340 USER_DEFINED = 441 UNUSED = 542 BYTE = 643 44 45AnyModel = TypeVar("AnyModel", bound="type[Model]")46 47 48class Model:49 _model_classes: dict[str, type[Model]] = {}50 51 dir_model: Path52 ftype: gguf.LlamaFileType53 fname_out: Path54 is_big_endian: bool55 endianess: gguf.GGUFEndian56 use_temp_file: bool57 lazy: bool58 part_names: list[str]59 is_safetensors: bool60 hparams: dict[str, Any]61 block_count: int62 tensor_map: gguf.TensorNameMap63 tensor_names: set[str] | None64 gguf_writer: gguf.GGUFWriter65 model_name: str | None66 metadata_override: Path | None67 dir_model_card: Path68 69 # subclasses should define this!70 model_arch: gguf.MODEL_ARCH71 72 def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, is_big_endian: bool = False,73 use_temp_file: bool = False, eager: bool = False,74 metadata_override: Path | None = None, model_name: str | None = None,75 split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False,76 small_first_shard: bool = False, hparams: dict[str, Any] | None = None):77 if type(self) is Model:78 raise TypeError(f"{type(self).__name__!r} should not be directly instantiated")79 80 self.dir_model = dir_model81 self.ftype = ftype82 self.fname_out = fname_out83 self.is_big_endian = is_big_endian84 self.endianess = gguf.GGUFEndian.BIG if is_big_endian else gguf.GGUFEndian.LITTLE85 self.use_temp_file = use_temp_file86 self.lazy = not eager87 self.part_names = Model.get_model_part_names(self.dir_model, "model", ".safetensors")88 self.is_safetensors = len(self.part_names) > 089 if not self.is_safetensors:90 self.part_names = Model.get_model_part_names(self.dir_model, "pytorch_model", ".bin")91 self.hparams = Model.load_hparams(self.dir_model) if hparams is None else hparams92 self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer", "num_layers"])93 self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)94 self.tensor_names = None95 self.metadata_override = metadata_override96 self.model_name = model_name97 self.dir_model_card = dir_model # overridden in convert_lora_to_gguf.py98 99 # Apply heuristics to figure out typical tensor encoding based on first layer tensor encoding type100 if self.ftype == gguf.LlamaFileType.GUESSED:101 # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.102 _, first_tensor = next(self.get_tensors())103 if first_tensor.dtype == torch.float16:104 logger.info(f"choosing --outtype f16 from first tensor type ({first_tensor.dtype})")105 self.ftype = gguf.LlamaFileType.MOSTLY_F16106 else:107 logger.info(f"choosing --outtype bf16 from first tensor type ({first_tensor.dtype})")108 self.ftype = gguf.LlamaFileType.MOSTLY_BF16109 110 # Configure GGUF Writer111 self.gguf_writer = gguf.GGUFWriter(path=None, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file,112 split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard)113 114 @classmethod115 def __init_subclass__(cls):116 # can't use an abstract property, because overriding it without type errors117 # would require using decorated functions instead of simply defining the property118 if "model_arch" not in cls.__dict__:119 raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")120 121 def find_hparam(self, keys: Iterable[str], optional: bool = False) -> Any:122 key = next((k for k in keys if k in self.hparams), None)123 if key is not None:124 return self.hparams[key]125 if optional:126 return None127 raise KeyError(f"could not find any of: {keys}")128 129 def set_vocab(self):130 self._set_vocab_gpt2()131 132 def get_tensors(self) -> Iterator[tuple[str, Tensor]]:133 tensor_names_from_parts: set[str] = set()134 135 index_name = "model.safetensors" if self.is_safetensors else "pytorch_model.bin"136 index_name += ".index.json"137 index_file = self.dir_model / index_name138 139 if index_file.is_file():140 self.tensor_names = set()141 logger.info(f"gguf: loading model weight map from '{index_name}'")142 with open(index_file, "r", encoding="utf-8") as f:143 index: dict[str, Any] = json.load(f)144 weight_map = index.get("weight_map")145 if weight_map is None or not isinstance(weight_map, dict):146 raise ValueError(f"Can't load 'weight_map' from {index_name!r}")147 self.tensor_names.update(weight_map.keys())148 else:149 self.tensor_names = tensor_names_from_parts150 weight_map = {}151 152 for part_name in self.part_names:153 logger.info(f"gguf: loading model part '{part_name}'")154 ctx: ContextManager[Any]155 if self.is_safetensors:156 from safetensors import safe_open157 ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu"))158 else:159 ctx = contextlib.nullcontext(torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True))160 161 with ctx as model_part:162 tensor_names_from_parts.update(model_part.keys())163 164 for name in model_part.keys():165 if self.is_safetensors:166 if self.lazy:167 data = model_part.get_slice(name)168 data = LazyTorchTensor.from_safetensors_slice(data)169 else:170 data = model_part.get_tensor(name)171 else:172 data = model_part[name]173 if self.lazy:174 data = LazyTorchTensor.from_eager(data)175 yield name, data176 177 # verify tensor name presence and identify potentially missing files178 if len(tensor_names_from_parts.symmetric_difference(self.tensor_names)) > 0:179 missing = sorted(self.tensor_names.difference(tensor_names_from_parts))180 extra = sorted(tensor_names_from_parts.difference(self.tensor_names))181 missing_files = sorted(set(weight_map[n] for n in missing if n in weight_map))182 if len(extra) == 0 and len(missing_files) > 0:183 raise ValueError(f"Missing or incomplete model files: {missing_files}")184 else:185 raise ValueError("Mismatch between weight map and model parts for tensor names:\n"186 f"Missing tensors: {missing}\n"187 f"Extra tensors: {extra}")188 189 def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:190 if key not in gguf.MODEL_TENSORS[self.model_arch]:191 raise ValueError(f"Missing {key!r} for MODEL_TENSORS of {self.model_arch!r}")192 name: str = gguf.TENSOR_NAMES[key]193 if "{bid}" in name:194 assert bid is not None195 name = name.format(bid=bid)196 return name + suffix197 198 def match_model_tensor_name(self, name: str, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> bool:199 if key not in gguf.MODEL_TENSORS[self.model_arch]:200 return False201 key_name: str = gguf.TENSOR_NAMES[key]202 if "{bid}" in key_name:203 if bid is None:204 return False205 key_name = key_name.format(bid=bid)206 else:207 if bid is not None:208 return False209 return name == (key_name + suffix)210 211 def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:212 new_name = self.tensor_map.get_name(key=name, try_suffixes=try_suffixes)213 if new_name is None:214 raise ValueError(f"Can not map tensor {name!r}")215 return new_name216 217 def set_gguf_parameters(self):218 self.gguf_writer.add_block_count(self.block_count)219 220 if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None:221 self.gguf_writer.add_context_length(n_ctx)222 logger.info(f"gguf: context length = {n_ctx}")223 224 if (n_embd := self.find_hparam(["hidden_size", "n_embd"], optional=True)) is not None:225 self.gguf_writer.add_embedding_length(n_embd)226 logger.info(f"gguf: embedding length = {n_embd}")227 228 if (n_ff := self.find_hparam(["intermediate_size", "n_inner"], optional=True)) is not None:229 self.gguf_writer.add_feed_forward_length(n_ff)230 logger.info(f"gguf: feed forward length = {n_ff}")231 232 if (n_head := self.find_hparam(["num_attention_heads", "n_head"], optional=True)) is not None:233 self.gguf_writer.add_head_count(n_head)234 logger.info(f"gguf: head count = {n_head}")235 236 if (n_head_kv := self.hparams.get("num_key_value_heads")) is not None:237 self.gguf_writer.add_head_count_kv(n_head_kv)238 logger.info(f"gguf: key-value head count = {n_head_kv}")239 240 if (rope_theta := self.hparams.get("rope_theta")) is not None:241 self.gguf_writer.add_rope_freq_base(rope_theta)242 logger.info(f"gguf: rope theta = {rope_theta}")243 if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None:244 self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)245 logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")246 if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None:247 self.gguf_writer.add_layer_norm_eps(f_norm_eps)248 logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")249 if (n_experts := self.hparams.get("num_local_experts")) is not None:250 self.gguf_writer.add_expert_count(n_experts)251 logger.info(f"gguf: expert count = {n_experts}")252 if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:253 self.gguf_writer.add_expert_used_count(n_experts_used)254 logger.info(f"gguf: experts used count = {n_experts_used}")255 256 if (head_dim := self.hparams.get("head_dim")) is not None:257 self.gguf_writer.add_key_length(head_dim)258 self.gguf_writer.add_value_length(head_dim)259 260 self.gguf_writer.add_file_type(self.ftype)261 logger.info(f"gguf: file type = {self.ftype}")262 263 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:264 del bid # unused265 266 return [(self.map_tensor_name(name), data_torch)]267 268 def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:269 del name, new_name, bid, n_dims # unused270 271 return False272 273 # some models need extra generated tensors (like rope_freqs)274 def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:275 return ()276 277 def prepare_tensors(self):278 max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")279 280 for name, data_torch in chain(self.generate_extra_tensors(), self.get_tensors()):281 # we don't need these282 if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):283 continue284 285 old_dtype = data_torch.dtype286 287 # convert any unsupported data types to float32288 if data_torch.dtype not in (torch.float16, torch.float32):289 data_torch = data_torch.to(torch.float32)290 291 # use the first number-like part of the tensor name as the block id292 bid = None293 for part in name.split("."):294 if part.isdecimal():295 bid = int(part)296 break297 298 for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):299 # TODO: why do we squeeze here?300 # data = data_torch.squeeze().numpy()301 data = data_torch.numpy()302 303 # if data ends up empty, it means data_torch was a scalar tensor -> restore304 if len(data.shape) == 0:305 data = data_torch.numpy()306 307 n_dims = len(data.shape)308 data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)309 310 # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors311 if n_dims <= 1 or new_name.endswith("_norm.weight"):312 data_qtype = gguf.GGMLQuantizationType.F32313 314 # Conditions should closely match those in llama_model_quantize_internal in llama.cpp315 # Some tensor types are always in float32316 if data_qtype is False and (317 any(318 self.match_model_tensor_name(new_name, key, bid)319 for key in (320 gguf.MODEL_TENSOR.FFN_GATE_INP,321 gguf.MODEL_TENSOR.POS_EMBD,322 gguf.MODEL_TENSOR.TOKEN_TYPES,323 gguf.MODEL_TENSOR.SSM_CONV1D,324 gguf.MODEL_TENSOR.TIME_MIX_FIRST,325 gguf.MODEL_TENSOR.TIME_MIX_W1,326 gguf.MODEL_TENSOR.TIME_MIX_W2,327 gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,328 gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,329 gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,330 gguf.MODEL_TENSOR.POSNET_NORM1,331 gguf.MODEL_TENSOR.POSNET_NORM2,332 )333 )334 or not new_name.endswith(".weight")335 ):336 data_qtype = gguf.GGMLQuantizationType.F32337 338 if data_qtype is False and any(339 self.match_model_tensor_name(new_name, key, bid)340 for key in (341 gguf.MODEL_TENSOR.TOKEN_EMBD,342 gguf.MODEL_TENSOR.OUTPUT,343 )344 ):345 if self.ftype in (346 gguf.LlamaFileType.MOSTLY_TQ1_0,347 gguf.LlamaFileType.MOSTLY_TQ2_0,348 ):349 # TODO: use Q4_K and Q6_K350 data_qtype = gguf.GGMLQuantizationType.F16351 352 # No override (data_qtype is False), or wants to be quantized (data_qtype is True)353 if isinstance(data_qtype, bool):354 if self.ftype == gguf.LlamaFileType.ALL_F32:355 data_qtype = gguf.GGMLQuantizationType.F32356 elif self.ftype == gguf.LlamaFileType.MOSTLY_F16:357 data_qtype = gguf.GGMLQuantizationType.F16358 elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:359 data_qtype = gguf.GGMLQuantizationType.BF16360 elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:361 data_qtype = gguf.GGMLQuantizationType.Q8_0362 elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ1_0:363 data_qtype = gguf.GGMLQuantizationType.TQ1_0364 elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ2_0:365 data_qtype = gguf.GGMLQuantizationType.TQ2_0366 else:367 raise ValueError(f"Unknown file type: {self.ftype.name}")368 369 try:370 data = gguf.quants.quantize(data, data_qtype)371 except gguf.QuantError as e:372 logger.warning("%s, %s", e, "falling back to F16")373 data_qtype = gguf.GGMLQuantizationType.F16374 data = gguf.quants.quantize(data, data_qtype)375 376 shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape377 378 # reverse shape to make it similar to the internal ggml dimension order379 shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"380 381 # n_dims is implicit in the shape382 logger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}")383 384 self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)385 386 def set_type(self):387 self.gguf_writer.add_type(gguf.GGUFType.MODEL)388 389 def prepare_metadata(self, vocab_only: bool):390 391 total_params, shared_params, expert_params, expert_count = self.gguf_writer.get_total_parameter_count()392 393 self.metadata = gguf.Metadata.load(self.metadata_override, self.dir_model_card, self.model_name, total_params)394 395 # Fallback to model directory name if metadata name is still missing396 if self.metadata.name is None:397 self.metadata.name = self.dir_model.name398 399 # Generate parameter weight class (useful for leader boards) if not yet determined400 if self.metadata.size_label is None and total_params > 0:401 self.metadata.size_label = gguf.size_label(total_params, shared_params, expert_params, expert_count)402 403 # Extract the encoding scheme from the file type name. e.g. 'gguf.LlamaFileType.MOSTLY_Q8_0' --> 'Q8_0'404 output_type: str = self.ftype.name.partition("_")[2]405 406 # Filename Output407 if self.fname_out.is_dir():408 # Generate default filename based on model specification and available metadata409 if not vocab_only:410 fname_default: str = gguf.naming_convention(self.metadata.name, self.metadata.basename, self.metadata.finetune, self.metadata.version, self.metadata.size_label, output_type, model_type="LoRA" if total_params < 0 else None)411 else:412 fname_default: str = gguf.naming_convention(self.metadata.name, self.metadata.basename, self.metadata.finetune, self.metadata.version, size_label=None, output_type=None, model_type="vocab")413 414 # Use the default filename415 self.fname_out = self.fname_out / f"{fname_default}.gguf"416 else:417 # Output path is a custom defined templated filename418 # Note: `not is_dir()` is used because `.is_file()` will not detect419 # file template strings as it doesn't actually exist as a file420 421 # Process templated file name with the output ftype, useful with the "auto" ftype422 self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type)423 424 self.set_type()425 426 logger.info("Set meta model")427 self.metadata.set_gguf_meta_model(self.gguf_writer)428 429 logger.info("Set model parameters")430 self.set_gguf_parameters()431 432 logger.info("Set model tokenizer")433 self.set_vocab()434 435 logger.info("Set model quantization version")436 self.gguf_writer.add_quantization_version(gguf.GGML_QUANT_VERSION)437 438 def write(self):439 self.prepare_tensors()440 self.prepare_metadata(vocab_only=False)441 self.gguf_writer.write_header_to_file(path=self.fname_out)442 self.gguf_writer.write_kv_data_to_file()443 self.gguf_writer.write_tensors_to_file(progress=True)444 self.gguf_writer.close()445 446 def write_vocab(self):447 if len(self.gguf_writer.tensors) != 1:448 raise ValueError('Splitting the vocabulary is not supported')449 450 self.prepare_metadata(vocab_only=True)451 self.gguf_writer.write_header_to_file(path=self.fname_out)452 self.gguf_writer.write_kv_data_to_file()453 self.gguf_writer.close()454 455 @staticmethod456 def get_model_part_names(dir_model: Path, prefix: str, suffix: str) -> list[str]:457 part_names: list[str] = []458 for filename in os.listdir(dir_model):459 if filename.startswith(prefix) and filename.endswith(suffix):460 part_names.append(filename)461 462 part_names.sort()463 464 return part_names465 466 @staticmethod467 def load_hparams(dir_model: Path):468 with open(dir_model / "config.json", "r", encoding="utf-8") as f:469 return json.load(f)470 471 @classmethod472 def register(cls, *names: str) -> Callable[[AnyModel], AnyModel]:473 assert names474 475 def func(modelcls: AnyModel) -> AnyModel:476 for name in names:477 cls._model_classes[name] = modelcls478 return modelcls479 return func480 481 @classmethod482 def print_registered_models(cls):483 for name in sorted(cls._model_classes.keys()):484 logger.error(f"- {name}")485 486 @classmethod487 def from_model_architecture(cls, arch: str) -> type[Model]:488 try:489 return cls._model_classes[arch]490 except KeyError:491 raise NotImplementedError(f'Architecture {arch!r} not supported!') from None492 493 def does_token_look_special(self, token: str | bytes) -> bool:494 if isinstance(token, (bytes, bytearray)):495 token_text = token.decode(encoding="utf-8")496 elif isinstance(token, memoryview):497 token_text = token.tobytes().decode(encoding="utf-8")498 else:499 token_text = token500 501 # Some models mark some added tokens which ought to be control tokens as not special.502 # (e.g. command-r, command-r-plus, deepseek-coder, gemma{,-2})503 seems_special = token_text in (504 "<pad>", # deepseek-coder505 "<mask>", "<2mass>", "[@BOS@]", # gemma{,-2}506 )507 508 seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))509 seems_special = seems_special or (token_text.startswith("<๏ฝ") and token_text.endswith("๏ฝ>")) # deepseek-coder510 511 # TODO: should these be marked as UNUSED instead? (maybe not)512 seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}513 514 return seems_special515 516 # used for GPT-2 BPE and WordPiece vocabs517 def get_vocab_base(self) -> tuple[list[str], list[int], str]:518 tokens: list[str] = []519 toktypes: list[int] = []520 521 from transformers import AutoTokenizer522 tokenizer = AutoTokenizer.from_pretrained(self.dir_model)523 vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))524 assert max(tokenizer.vocab.values()) < vocab_size525 526 tokpre = self.get_vocab_base_pre(tokenizer)527 528 reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}529 added_vocab = tokenizer.get_added_vocab()530 531 for i in range(vocab_size):532 if i not in reverse_vocab:533 tokens.append(f"[PAD{i}]")534 toktypes.append(gguf.TokenType.UNUSED)535 else:536 token: str = reverse_vocab[i]537 if token in added_vocab:538 # The tokenizer in llama.cpp assumes the CONTROL and USER_DEFINED tokens are pre-normalized.539 # To avoid unexpected issues - we make sure to normalize non-normalized tokens540 if not tokenizer.added_tokens_decoder[i].normalized:541 previous_token = token542 token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))543 if previous_token != token:544 logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")545 546 if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token):547 toktypes.append(gguf.TokenType.CONTROL)548 else:549 # NOTE: this was added for Gemma.550 # Encoding and decoding the tokens above isn't sufficient for this case.551 token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces552 toktypes.append(gguf.TokenType.USER_DEFINED)553 else:554 toktypes.append(gguf.TokenType.NORMAL)555 tokens.append(token)556 557 return tokens, toktypes, tokpre558 559 # NOTE: this function is generated by convert_hf_to_gguf_update.py560 # do not modify it manually!561 # ref: https://github.com/ggerganov/llama.cpp/pull/6920562 # Marker: Start get_vocab_base_pre563 def get_vocab_base_pre(self, tokenizer) -> str:564 # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that565 # is specific for the BPE pre-tokenizer used by the model566 # we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can567 # use in llama.cpp to implement the same pre-tokenizer568 569 chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n๐ (normal) ๐ถ\u200d๐ซ๏ธ (multiple emojis concatenated) โ
๐ฆ๐ฆ 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 แแถแแแแแแทแแแแขแถแ
๐ ?ๆๆณๅจappleๅทฅไฝ1314151ๅคฉ๏ฝ ------======= ะฝะตัะพ ะฝะฐ ะัะปะณะฐััะบะธ \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'570 571 chktok = tokenizer.encode(chktxt)572 chkhsh = sha256(str(chktok).encode()).hexdigest()573 574 logger.debug(f"chktok: {chktok}")575 logger.debug(f"chkhsh: {chkhsh}")576 577 res = None578 579 # NOTE: if you get an error here, you need to update the convert_hf_to_gguf_update.py script580 # or pull the latest version of the model from Huggingface581 # don't edit the hashes manually!582 if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":583 # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B584 res = "llama-bpe"585 if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754":586 # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base587 res = "deepseek-llm"588 if chkhsh == "347715f544604f9118bb75ed199f68779f423cabb20db6de6f31b908d04d7821":589 # ref: https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base590 res = "deepseek-coder"591 if chkhsh == "8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652ed":592 # ref: https://huggingface.co/tiiuae/falcon-7b593 res = "falcon"594 if chkhsh == "9d032fcbd5501f4a38150912590928bfb36091efb5df11b8e2124b0390e3fb1e":595 # ref: https://huggingface.co/tiiuae/Falcon3-7B-Base596 res = "falcon3"597 if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":598 # ref: https://huggingface.co/BAAI/bge-small-en-v1.5599 res = "bert-bge"600 if chkhsh == "8e62295832751ca1e8f92f2226f403dea30dc5165e448b5bfa05af5340c64ec7":601 # ref: https://huggingface.co/BAAI/bge-large-zh-v1.5602 res = "bert-bge-large"603 if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166":604 # ref: https://huggingface.co/mosaicml/mpt-7b605 res = "mpt"606 if chkhsh == "35d91631860c815f952d711435f48d356ebac988362536bed955d43bfa436e34":607 # ref: https://huggingface.co/bigcode/starcoder2-3b608 res = "starcoder"609 if chkhsh == "3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454":610 # ref: https://huggingface.co/openai-community/gpt2611 res = "gpt-2"612 if chkhsh == "32d85c31273f8019248f2559fed492d929ea28b17e51d81d3bb36fff23ca72b3":613 # ref: https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b614 res = "stablelm2"615 if chkhsh == "6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ff":616 # ref: https://huggingface.co/smallcloudai/Refact-1_6-base617 res = "refact"618 if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8":619 # ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01620 res = "command-r"621 if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":622 # ref: https://huggingface.co/Qwen/Qwen1.5-7B623 res = "qwen2"624 if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166":625 # ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf626 res = "olmo"627 if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e":628 # ref: https://huggingface.co/databricks/dbrx-base629 res = "dbrx"630 if chkhsh == "c7699093ba4255a91e702aa38a596aa81669f3525dae06c2953267dde580f448":631 # ref: https://huggingface.co/jinaai/jina-reranker-v1-tiny-en632 res = "jina-v1-en"633 if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":634 # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en635 res = "jina-v2-en"636 if chkhsh == "171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643":637 # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-es638 res = "jina-v2-es"639 if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6":640 # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de641 res = "jina-v2-de"642 if chkhsh == "c136ed14d01c2745d4f60a9596ae66800e2b61fa45643e72436041855ad4089d":643 # ref: https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct644 res = "smaug-bpe"645 if chkhsh == "c7ea5862a53e4272c035c8238367063e2b270d51faa48c0f09e9d5b54746c360":646 # ref: https://huggingface.co/LumiOpen/Poro-34B-chat647 res = "poro-chat"648 if chkhsh == "7967bfa498ade6b757b064f31e964dddbb80f8f9a4d68d4ba7998fcf281c531a":649 # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-code650 res = "jina-v2-code"651 if chkhsh == "b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99b" or chkhsh == "81d72c7348a9f0ebe86f23298d37debe0a5e71149e29bd283904c02262b27516":652 # ref: https://huggingface.co/THUDM/glm-4-9b-chat653 res = "chatglm-bpe"654 if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee":655 # ref: https://huggingface.co/LumiOpen/Viking-7B656 res = "viking"657 if chkhsh == "b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901":658 # ref: https://huggingface.co/core42/jais-13b659 res = "jais"660 if chkhsh == "7b3e7548e4308f52a76e8229e4e6cc831195d0d1df43aed21ac6c93da05fec5f":661 # ref: https://huggingface.co/WisdomShell/CodeShell-7B662 res = "codeshell"663 if chkhsh == "63b97e4253352e6f357cc59ea5b583e3a680eaeaf2632188c2b952de2588485e":664 # ref: https://huggingface.co/mistralai/Mistral-Nemo-Base-2407665 res = "tekken"666 if chkhsh == "855059429035d75a914d1eda9f10a876752e281a054a7a3d421ef0533e5b6249":667 # ref: https://huggingface.co/HuggingFaceTB/SmolLM-135M668 res = "smollm"669 if chkhsh == "3c30d3ad1d6b64202cd222813e7736c2db6e1bd6d67197090fc1211fbc612ae7":670 # ref: https://huggingface.co/bigscience/bloom671 res = "bloom"672 if chkhsh == "bc01ce58980e1db43859146dc51b1758b3b88729b217a74792e9f8d43e479d21":673 # ref: https://huggingface.co/TurkuNLP/gpt3-finnish-small674 res = "gpt3-finnish"675 if chkhsh == "4e2b24cc4770243d65a2c9ec19770a72f08cffc161adbb73fcbb6b7dd45a0aae":676 # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct677 res = "exaone"678 if chkhsh == "fcace8b9cac38ce847670c970cd5892031a753a1ef381abd1d9af00f713da085":679 # ref: https://huggingface.co/microsoft/phi-2680 res = "phi-2"681 if chkhsh == "60824e3c0d9401f89943cbb2fff727f0e2d4c545ba4df2d6e4f09a6db0f5b450":682 # ref: https://huggingface.co/facebook/chameleon-7b683 res = "chameleon"684 if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":685 # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0686 res = "minerva-7b"687 if chkhsh == "8b5a93ed704057481f240da0be7e7dca721d7f8f4755263b6807227a2cbeae65":688 # ref: https://huggingface.co/sentence-transformers/stsb-roberta-base689 res = "roberta-bpe"690 if chkhsh == "ad851be1dba641f2e3711822f816db2c265f788b37c63b4e1aeacb9ee92de8eb":691 # ref: https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct692 res = "gigachat"693 if chkhsh == "d4c8f286ea6b520b3d495c4455483cfa2302c0cfcd4be05d781b6a8a0a7cdaf1":694 # ref: https://huggingface.co/Infinigence/Megrez-3B-Instruct695 res = "megrez"696 if chkhsh == "877081d19cf6996e2c4ff0e1236341e9b7bde288f5311a56a937f0afbbb3aeb5":697 # ref: https://huggingface.co/deepseek-ai/DeepSeek-V3698 res = "deepseek-v3"699 if chkhsh == "b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5":700 # ref: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B701 res = "deepseek-r1-qwen"702 703 if res is None:704 logger.warning("\n")705 logger.warning("**************************************************************************************")706 logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")707 logger.warning("** There are 2 possible reasons for this:")708 logger.warning("** - the model has not been added to convert_hf_to_gguf_update.py yet")709 logger.warning("** - the pre-tokenization config has changed upstream")710 logger.warning("** Check your model files and convert_hf_to_gguf_update.py and update them accordingly.")711 logger.warning("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")712 logger.warning("**")713 logger.warning(f"** chkhsh: {chkhsh}")714 logger.warning("**************************************************************************************")715 logger.warning("\n")716 raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")717 718 logger.debug(f"tokenizer.ggml.pre: {repr(res)}")719 logger.debug(f"chkhsh: {chkhsh}")720 721 return res722 # Marker: End get_vocab_base_pre723 724 def _set_vocab_none(self) -> None:725 self.gguf_writer.add_tokenizer_model("none")726 727 def _set_vocab_gpt2(self) -> None:728 tokens, toktypes, tokpre = self.get_vocab_base()729 self.gguf_writer.add_tokenizer_model("gpt2")730 self.gguf_writer.add_tokenizer_pre(tokpre)731 self.gguf_writer.add_token_list(tokens)732 self.gguf_writer.add_token_types(toktypes)733 734 special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)735 special_vocab.add_to_gguf(self.gguf_writer)736 737 def _set_vocab_qwen(self):738 dir_model = self.dir_model739 hparams = self.hparams740 tokens: list[str] = []741 toktypes: list[int] = []742 743 from transformers import AutoTokenizer744 tokenizer = AutoTokenizer.from_pretrained(dir_model, trust_remote_code=True)745 vocab_size = hparams["vocab_size"]746 assert max(tokenizer.get_vocab().values()) < vocab_size747 748 tokpre = self.get_vocab_base_pre(tokenizer)749 750 merges = []751 vocab = {}752 mergeable_ranks = tokenizer.mergeable_ranks753 for token, rank in mergeable_ranks.items():754 vocab[QwenModel.token_bytes_to_string(token)] = rank755 if len(token) == 1:756 continue757 merged = QwenModel.bpe(mergeable_ranks, token, max_rank=rank)758 assert len(merged) == 2759 merges.append(' '.join(map(QwenModel.token_bytes_to_string, merged)))760 761 # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined762 added_vocab = tokenizer.special_tokens763 reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()}764 765 for i in range(vocab_size):766 if i not in reverse_vocab:767 tokens.append(f"[PAD{i}]")768 toktypes.append(gguf.TokenType.UNUSED)769 elif reverse_vocab[i] in added_vocab:770 tokens.append(reverse_vocab[i])771 toktypes.append(gguf.TokenType.CONTROL)772 else:773 tokens.append(reverse_vocab[i])774 toktypes.append(gguf.TokenType.NORMAL)775 776 self.gguf_writer.add_tokenizer_model("gpt2")777 self.gguf_writer.add_tokenizer_pre(tokpre)778 self.gguf_writer.add_token_list(tokens)779 self.gguf_writer.add_token_types(toktypes)780 781 special_vocab = gguf.SpecialVocab(dir_model, load_merges=False)782 special_vocab.merges = merges783 # only add special tokens when they were not already loaded from config.json784 if len(special_vocab.special_token_ids) == 0:785 special_vocab._set_special_token("bos", tokenizer.special_tokens["<|endoftext|>"])786 special_vocab._set_special_token("eos", tokenizer.special_tokens["<|endoftext|>"])787 # this one is usually not in config.json anyway788 special_vocab._set_special_token("unk", tokenizer.special_tokens["<|endoftext|>"])789 special_vocab.add_to_gguf(self.gguf_writer)790 791 def _set_vocab_sentencepiece(self, add_to_gguf=True):792 tokens, scores, toktypes = self._create_vocab_sentencepiece()793 794 self.gguf_writer.add_tokenizer_model("llama")795 self.gguf_writer.add_tokenizer_pre("default")796 self.gguf_writer.add_token_list(tokens)797 self.gguf_writer.add_token_scores(scores)798 self.gguf_writer.add_token_types(toktypes)799 800 special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))801 special_vocab.add_to_gguf(self.gguf_writer)802 803 def _create_vocab_sentencepiece(self):804 from sentencepiece import SentencePieceProcessor805 806 tokenizer_path = self.dir_model / 'tokenizer.model'807 808 if not tokenizer_path.is_file():809 raise FileNotFoundError(f"File not found: {tokenizer_path}")810 811 tokenizer = SentencePieceProcessor()812 tokenizer.LoadFromFile(str(tokenizer_path))813 814 vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size())815 816 tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)]817 scores: list[float] = [-10000.0] * vocab_size818 toktypes: list[int] = [SentencePieceTokenTypes.UNUSED] * vocab_size819 820 for token_id in range(tokenizer.vocab_size()):821 piece = tokenizer.IdToPiece(token_id)822 text = piece.encode("utf-8")823 score = tokenizer.GetScore(token_id)824 825 toktype = SentencePieceTokenTypes.NORMAL826 if tokenizer.IsUnknown(token_id):827 toktype = SentencePieceTokenTypes.UNKNOWN828 elif tokenizer.IsControl(token_id):829 toktype = SentencePieceTokenTypes.CONTROL830 elif tokenizer.IsUnused(token_id):831 toktype = SentencePieceTokenTypes.UNUSED832 elif tokenizer.IsByte(token_id):833 toktype = SentencePieceTokenTypes.BYTE834 835 tokens[token_id] = text836 scores[token_id] = score837 toktypes[token_id] = toktype838 839 added_tokens_file = self.dir_model / 'added_tokens.json'840 if added_tokens_file.is_file():841 with open(added_tokens_file, "r", encoding="utf-8") as f:842 added_tokens_json = json.load(f)843 for key in added_tokens_json:844 token_id = added_tokens_json[key]845 if token_id >= vocab_size:846 logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')847 continue848 849 tokens[token_id] = key.encode("utf-8")850 scores[token_id] = -1000.0851 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED852 853 tokenizer_config_file = self.dir_model / 'tokenizer_config.json'854 if tokenizer_config_file.is_file():855 with open(tokenizer_config_file, "r", encoding="utf-8") as f:856 tokenizer_config_json = json.load(f)857 added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {})858 for token_id, token_data in added_tokens_decoder.items():859 token_id = int(token_id)860 token: str = token_data["content"]861 if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:862 if tokens[token_id] != token.encode("utf-8"):863 logger.warning(f'replacing token {token_id}: {tokens[token_id].decode("utf-8")!r} -> {token!r}')864 if token_data.get("special") or self.does_token_look_special(token):865 toktypes[token_id] = SentencePieceTokenTypes.CONTROL866 else:867 token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces868 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED869 870 scores[token_id] = -1000.0871 tokens[token_id] = token.encode("utf-8")872 873 if vocab_size > len(tokens):874 pad_count = vocab_size - len(tokens)875 logger.debug(f"Padding vocab with {pad_count} token(s) - [PAD1] through [PAD{pad_count}]")876 for i in range(1, pad_count + 1):877 tokens.append(bytes(f"[PAD{i}]", encoding="utf-8"))878 scores.append(-1000.0)879 toktypes.append(SentencePieceTokenTypes.UNUSED)880 881 return tokens, scores, toktypes882 883 def _set_vocab_llama_hf(self):884 vocab = gguf.LlamaHfVocab(self.dir_model)885 tokens = []886 scores = []887 toktypes = []888 889 for text, score, toktype in vocab.all_tokens():890 tokens.append(text)891 scores.append(score)892 toktypes.append(toktype)893 894 assert len(tokens) == vocab.vocab_size895 896 self.gguf_writer.add_tokenizer_model("llama")897 self.gguf_writer.add_tokenizer_pre("default")898 self.gguf_writer.add_token_list(tokens)899 self.gguf_writer.add_token_scores(scores)900 self.gguf_writer.add_token_types(toktypes)901 902 special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))903 special_vocab.add_to_gguf(self.gguf_writer)904 905 def _set_vocab_builtin(self, model_name: Literal["gpt-neox", "llama-spm"], vocab_size: int):906 tokenizer_path = Path(sys.path[0]) / "models" / f"ggml-vocab-{model_name}.gguf"907 logger.warning(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'")908 vocab_reader = gguf.GGUFReader(tokenizer_path, "r")909 910 default_pre = "mpt" if model_name == "gpt-neox" else "default"911 912 field = vocab_reader.get_field(gguf.Keys.Tokenizer.MODEL)913 assert field # tokenizer model914 self.gguf_writer.add_tokenizer_model(bytes(field.parts[-1]).decode("utf-8"))915 916 field = vocab_reader.get_field(gguf.Keys.Tokenizer.PRE)917 self.gguf_writer.add_tokenizer_pre(bytes(field.parts[-1]).decode("utf-8") if field else default_pre)918 919 field = vocab_reader.get_field(gguf.Keys.Tokenizer.LIST)920 assert field # token list921 self.gguf_writer.add_token_list([bytes(field.parts[i]) for i in field.data][:vocab_size])922 923 if model_name == "llama-spm":924 field = vocab_reader.get_field(gguf.Keys.Tokenizer.SCORES)925 assert field # token scores926 self.gguf_writer.add_token_scores([field.parts[i].tolist()[0] for i in field.data][:vocab_size])927 928 field = vocab_reader.get_field(gguf.Keys.Tokenizer.TOKEN_TYPE)929 assert field # token types930 self.gguf_writer.add_token_types([field.parts[i].tolist()[0] for i in field.data][:vocab_size])931 932 if model_name != "llama-spm":933 field = vocab_reader.get_field(gguf.Keys.Tokenizer.MERGES)934 assert field # token merges935 self.gguf_writer.add_token_merges([bytes(field.parts[i]) for i in field.data])936 937 if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.BOS_ID)) is not None:938 self.gguf_writer.add_bos_token_id(field.parts[-1].tolist()[0])939 if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.EOS_ID)) is not None:940 self.gguf_writer.add_eos_token_id(field.parts[-1].tolist()[0])941 if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.UNK_ID)) is not None:942 self.gguf_writer.add_unk_token_id(field.parts[-1].tolist()[0])943 if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.PAD_ID)) is not None:944 self.gguf_writer.add_pad_token_id(field.parts[-1].tolist()[0])945 if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_BOS)) is not None:946 self.gguf_writer.add_add_bos_token(field.parts[-1].tolist()[0])947 if (field := vocab_reader.get_field(gguf.Keys.Tokenizer.ADD_EOS)) is not None:948 self.gguf_writer.add_add_eos_token(field.parts[-1].tolist()[0])949 950 951@Model.register("GPTNeoXForCausalLM")952class GPTNeoXModel(Model):953 model_arch = gguf.MODEL_ARCH.GPTNEOX954 955 def set_gguf_parameters(self):956 block_count = self.hparams["num_hidden_layers"]957 958 self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"])959 self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])960 self.gguf_writer.add_block_count(block_count)961 self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])962 self.gguf_writer.add_rope_dimension_count(963 int(self.hparams["rotary_pct"] * (self.hparams["hidden_size"] // self.hparams["num_attention_heads"])),964 )965 self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])966 self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True))967 self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"])968 969 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:970 del bid # unused971 972 n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))973 n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))974 975 tensors: list[tuple[str, Tensor]] = []976 977 if re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.weight", name):978 # Map bloom-style qkv_linear to gpt-style qkv_linear979 # bloom: https://github.com/huggingface/transformers/blob/main/src/transformers/models/bloom/modeling_bloom.py#L238-L252 # noqa980 # gpt-2: https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py#L312 # noqa981 qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))982 data_torch = torch.cat(983 (984 qkv_weights[:, 0, :, :].reshape((-1, n_embed)),985 qkv_weights[:, 1, :, :].reshape((-1, n_embed)),986 qkv_weights[:, 2, :, :].reshape((-1, n_embed)),987 ),988 dim=0,989 )990 logger.info("re-format attention.linear_qkv.weight")991 elif re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias", name):992 qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head))993 data_torch = torch.cat(994 (995 qkv_bias[:, 0, :].reshape((n_embed,)),996 qkv_bias[:, 1, :].reshape((n_embed,)),997 qkv_bias[:, 2, :].reshape((n_embed,)),998 ),999 dim=0,1000 )1001 logger.info("re-format attention.linear_qkv.bias")1002 1003 tensors.append((self.map_tensor_name(name), data_torch))1004 1005 return tensors1006 1007 1008@Model.register("BloomForCausalLM", "BloomModel")1009class BloomModel(Model):1010 model_arch = gguf.MODEL_ARCH.BLOOM1011 1012 def set_gguf_parameters(self):1013 n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))1014 n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))1015 self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))1016 self.gguf_writer.add_embedding_length(n_embed)1017 self.gguf_writer.add_feed_forward_length(4 * n_embed)1018 self.gguf_writer.add_block_count(self.hparams["n_layer"])1019 self.gguf_writer.add_head_count(n_head)1020 self.gguf_writer.add_head_count_kv(n_head)1021 self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])1022 self.gguf_writer.add_file_type(self.ftype)1023 1024 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:1025 del bid # unused1026 1027 n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads"))1028 n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed"))1029 1030 name = re.sub(r'transformer\.', '', name)1031 1032 tensors: list[tuple[str, Tensor]] = []1033 1034 if re.match(r"h\.\d+\.self_attention\.query_key_value\.weight", name):1035 # Map bloom-style qkv_linear to gpt-style qkv_linear1036 # bloom: https://github.com/huggingface/transformers/blob/main/src/transformers/models/bloom/modeling_bloom.py#L238-L252 # noqa1037 # gpt-2: https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py#L312 # noqa1038 qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed))1039 data_torch = torch.cat(1040 (1041 qkv_weights[:, 0, :, :].reshape((-1, n_embed)),1042 qkv_weights[:, 1, :, :].reshape((-1, n_embed)),1043 qkv_weights[:, 2, :, :].reshape((-1, n_embed)),1044 ),1045 dim=0,1046 )1047 logger.info("re-format attention.linear_qkv.weight")1048 elif re.match(r"h\.\d+\.self_attention\.query_key_value\.bias", name):1049 qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head))1050 data_torch = torch.cat(1051 (1052 qkv_bias[:, 0, :].reshape((n_embed,)),1053 qkv_bias[:, 1, :].reshape((n_embed,)),1054 qkv_bias[:, 2, :].reshape((n_embed,)),1055 ),1056 dim=0,1057 )1058 logger.info("re-format attention.linear_qkv.bias")1059 1060 tensors.append((self.map_tensor_name(name), data_torch))1061 1062 if name == "word_embeddings.weight":1063 assert self.tensor_names is not None1064 1065 # TODO: tie them at runtime, don't duplicate in the model file1066 if all(s not in self.tensor_names for s in ("lm_head.weight", "output.weight")):1067 tensors.append((self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch))1068 1069 return tensors1070 1071 1072@Model.register("MPTForCausalLM")1073class MPTModel(Model):1074 model_arch = gguf.MODEL_ARCH.MPT1075 1076 def set_vocab(self):1077 try:1078 self._set_vocab_gpt2()1079 except Exception:1080 # Fallback for SEA-LION model1081 self._set_vocab_sentencepiece()1082 self.gguf_writer.add_add_bos_token(False)1083 self.gguf_writer.add_pad_token_id(3)1084 self.gguf_writer.add_eos_token_id(1)1085 self.gguf_writer.add_unk_token_id(0)1086 1087 def set_gguf_parameters(self):1088 block_count = self.hparams["n_layers"]1089 self.gguf_writer.add_context_length(self.hparams["max_seq_len"])1090 self.gguf_writer.add_embedding_length(self.hparams["d_model"])1091 self.gguf_writer.add_block_count(block_count)1092 self.gguf_writer.add_feed_forward_length(4 * self.hparams["d_model"])1093 self.gguf_writer.add_head_count(self.hparams["n_heads"])1094 if kv_n_heads := self.hparams["attn_config"].get("kv_n_heads"):1095 self.gguf_writer.add_head_count_kv(kv_n_heads)1096 self.gguf_writer.add_layer_norm_eps(1e-5)1097 if self.hparams["attn_config"]["clip_qkv"] is not None:1098 self.gguf_writer.add_clamp_kqv(self.hparams["attn_config"]["clip_qkv"])1099 if self.hparams["attn_config"]["alibi"]:1100 self.gguf_writer.add_max_alibi_bias(self.hparams["attn_config"]["alibi_bias_max"])1101 else:1102 self.gguf_writer.add_max_alibi_bias(0.0)1103 1104 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:1105 del bid # unused1106 1107 if "scales" in name:1108 new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias", ".scales"))1109 new_name = new_name.replace("scales", "act.scales")1110 else:1111 new_name = self.map_tensor_name(name, try_suffixes=(".weight", ".bias"))1112 1113 return [(new_name, data_torch)]1114 1115 1116@Model.register("OrionForCausalLM")1117class OrionModel(Model):1118 model_arch = gguf.MODEL_ARCH.ORION1119 1120 def set_vocab(self):1121 self._set_vocab_sentencepiece()1122 1123 def set_gguf_parameters(self):1124 block_count = self.hparams["num_hidden_layers"]1125 head_count = self.hparams["num_attention_heads"]1126 head_count_kv = self.hparams.get("num_key_value_heads", head_count)1127 1128 ctx_length = 01129 if "max_sequence_length" in self.hparams:1130 ctx_length = self.hparams["max_sequence_length"]1131 elif "max_position_embeddings" in self.hparams:1132 ctx_length = self.hparams["max_position_embeddings"]1133 elif "model_max_length" in self.hparams:1134 ctx_length = self.hparams["model_max_length"]1135 else:1136 raise ValueError("gguf: can not find ctx length parameter.")1137 1138 self.gguf_writer.add_file_type(self.ftype)1139 self.gguf_writer.add_tensor_data_layout("Meta AI original pth")1140 self.gguf_writer.add_context_length(ctx_length)1141 self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])1142 self.gguf_writer.add_block_count(block_count)1143 self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])1144 self.gguf_writer.add_head_count(head_count)1145 self.gguf_writer.add_head_count_kv(head_count_kv)1146 # note: config provides rms norm but it is actually layer norm1147 # ref: https://huggingface.co/OrionStarAI/Orion-14B-Chat/blob/276a17221ce42beb45f66fac657a41540e71f4f5/modeling_orion.py#L570-L5711148 self.gguf_writer.add_layer_norm_eps(self.hparams["rms_norm_eps"])1149 1150 1151@Model.register("BaichuanForCausalLM", "BaiChuanForCausalLM")1152class BaichuanModel(Model):1153 model_arch = gguf.MODEL_ARCH.BAICHUAN1154 1155 def set_vocab(self):1156 self._set_vocab_sentencepiece()1157 1158 def set_gguf_parameters(self):1159 block_count = self.hparams["num_hidden_layers"]1160 head_count = self.hparams["num_attention_heads"]1161 head_count_kv = self.hparams.get("num_key_value_heads", head_count)1162 1163 ctx_length = 01164 if "max_sequence_length" in self.hparams:1165 ctx_length = self.hparams["max_sequence_length"]1166 elif "max_position_embeddings" in self.hparams:1167 ctx_length = self.hparams["max_position_embeddings"]1168 elif "model_max_length" in self.hparams:1169 ctx_length = self.hparams["model_max_length"]1170 else:1171 raise ValueError("gguf: can not find ctx length parameter.")1172 1173 self.gguf_writer.add_tensor_data_layout("Meta AI original pth")1174 self.gguf_writer.add_context_length(ctx_length)1175 self.gguf_writer.add_embedding_length(self.hparams["hidden_size"])1176 self.gguf_writer.add_block_count(block_count)1177 self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])1178 self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"])1179 self.gguf_writer.add_head_count(head_count)1180 self.gguf_writer.add_head_count_kv(head_count_kv)1181 self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"])1182 self.gguf_writer.add_file_type(self.ftype)1183 1184 if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]:1185 if self.hparams["rope_scaling"].get("type") == "linear":1186 self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)1187 self.gguf_writer.add_rope_scaling_factor(self.hparams["rope_scaling"]["factor"])1188 1189 def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:1190 head_count = self.hparams["num_attention_heads"]1191 head_count_kv = self.hparams.get("num_key_value_heads", head_count)1192 1193 tensors: list[tuple[str, Tensor]] = []1194 1195 if bid is not None and name == f"model.layers.{bid}.self_attn.W_pack.weight":1196 logger.info(f"Unpacking and permuting layer {bid}")1197 tensors = [1198 (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid),1199 self._reverse_hf_permute_part(data_torch, 0, head_count, head_count)),1200 (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid),