KBaba7/llama.cpp
0
1#ifndef LLAMA_H2#define LLAMA_H3 4#include "ggml.h"5#include "ggml-cpu.h"6#include "ggml-backend.h"7 8#include <stddef.h>9#include <stdint.h>10#include <stdio.h>11#include <stdbool.h>12 13#ifdef LLAMA_SHARED14# if defined(_WIN32) && !defined(__MINGW32__)15# ifdef LLAMA_BUILD16# define LLAMA_API __declspec(dllexport)17# else18# define LLAMA_API __declspec(dllimport)19# endif20# else21# define LLAMA_API __attribute__ ((visibility ("default")))22# endif23#else24# define LLAMA_API25#endif26 27#ifdef __GNUC__28# define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))29#elif defined(_MSC_VER)30# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func31#else32# define DEPRECATED(func, hint) func33#endif34 35#define LLAMA_DEFAULT_SEED 0xFFFFFFFF36 37#define LLAMA_TOKEN_NULL -138 39#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'40#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'41#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'42 43#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN44#define LLAMA_SESSION_VERSION 945 46#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ47#define LLAMA_STATE_SEQ_VERSION 248 49#ifdef __cplusplus50extern "C" {51#endif52 53 //54 // C interface55 //56 // TODO: show sample usage57 //58 59 struct llama_vocab;60 struct llama_model;61 struct llama_context;62 struct llama_sampler;63 64 typedef int32_t llama_pos;65 typedef int32_t llama_token;66 typedef int32_t llama_seq_id;67 68 enum llama_vocab_type {69 LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab70 LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback71 LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE72 LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece73 LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram74 LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization75 };76 77 // pre-tokenization types78 enum llama_vocab_pre_type {79 LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,80 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,81 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,82 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,83 LLAMA_VOCAB_PRE_TYPE_FALCON = 4,84 LLAMA_VOCAB_PRE_TYPE_MPT = 5,85 LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,86 LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,87 LLAMA_VOCAB_PRE_TYPE_REFACT = 8,88 LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9,89 LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10,90 LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,91 LLAMA_VOCAB_PRE_TYPE_OLMO = 12,92 LLAMA_VOCAB_PRE_TYPE_DBRX = 13,93 LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,94 LLAMA_VOCAB_PRE_TYPE_PORO = 15,95 LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16,96 LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17,97 LLAMA_VOCAB_PRE_TYPE_VIKING = 18,98 LLAMA_VOCAB_PRE_TYPE_JAIS = 19,99 LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,100 LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,101 LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,102 LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,103 LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,104 LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,105 LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26,106 LLAMA_VOCAB_PRE_TYPE_MINERVA = 27,107 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28,108 };109 110 enum llama_rope_type {111 LLAMA_ROPE_TYPE_NONE = -1,112 LLAMA_ROPE_TYPE_NORM = 0,113 LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,114 LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,115 LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,116 };117 118 enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file119 LLAMA_TOKEN_TYPE_UNDEFINED = 0,120 LLAMA_TOKEN_TYPE_NORMAL = 1,121 LLAMA_TOKEN_TYPE_UNKNOWN = 2,122 LLAMA_TOKEN_TYPE_CONTROL = 3,123 LLAMA_TOKEN_TYPE_USER_DEFINED = 4,124 LLAMA_TOKEN_TYPE_UNUSED = 5,125 LLAMA_TOKEN_TYPE_BYTE = 6,126 };127 128 enum llama_token_attr {129 LLAMA_TOKEN_ATTR_UNDEFINED = 0,130 LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,131 LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,132 LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,133 LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?134 LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,135 LLAMA_TOKEN_ATTR_BYTE = 1 << 5,136 LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,137 LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,138 LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,139 LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,140 };141 142 // model file types143 enum llama_ftype {144 LLAMA_FTYPE_ALL_F32 = 0,145 LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors146 LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors147 LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors148 // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16149 // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed150 // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed151 LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors152 LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors153 LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors154 LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors155 LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors156 LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors157 LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors158 LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors159 LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors160 LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors161 LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors162 LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors163 LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors164 LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors165 LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors166 LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors167 LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors168 LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors169 LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors170 LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors171 LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors172 LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors173 LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors174 LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors175 LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors176 LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors177 //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack178 //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack179 //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack180 LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors181 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors182 183 LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file184 };185 186 enum llama_rope_scaling_type {187 LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,188 LLAMA_ROPE_SCALING_TYPE_NONE = 0,189 LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,190 LLAMA_ROPE_SCALING_TYPE_YARN = 2,191 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3,192 LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE,193 };194 195 enum llama_pooling_type {196 LLAMA_POOLING_TYPE_UNSPECIFIED = -1,197 LLAMA_POOLING_TYPE_NONE = 0,198 LLAMA_POOLING_TYPE_MEAN = 1,199 LLAMA_POOLING_TYPE_CLS = 2,200 LLAMA_POOLING_TYPE_LAST = 3,201 LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph202 };203 204 enum llama_attention_type {205 LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,206 LLAMA_ATTENTION_TYPE_CAUSAL = 0,207 LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,208 };209 210 enum llama_split_mode {211 LLAMA_SPLIT_MODE_NONE = 0, // single GPU212 LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs213 LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported214 };215 216 // TODO: simplify (https://github.com/ggerganov/llama.cpp/pull/9294#pullrequestreview-2286561979)217 typedef struct llama_token_data {218 llama_token id; // token id219 float logit; // log-odds of the token220 float p; // probability of the token221 } llama_token_data;222 223 typedef struct llama_token_data_array {224 // TODO: consider SoA225 // NOTE: this pointer can be modified by the samplers226 llama_token_data * data;227 size_t size;228 int64_t selected; // this is the index in the data array (i.e. not the token id)229 bool sorted;230 } llama_token_data_array;231 232 typedef bool (*llama_progress_callback)(float progress, void * user_data);233 234 // Input data for llama_decode235 // A llama_batch object can contain input about one or many sequences236 // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens237 //238 // - token : the token ids of the input (used when embd is NULL)239 // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)240 // - pos : the positions of the respective token in the sequence241 // (if set to NULL, the token position will be tracked automatically by llama_decode)242 // - seq_id : the sequence to which the respective token belongs243 // (if set to NULL, the sequence ID will be assumed to be 0)244 // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output245 // (if set to NULL, only the logits for last token will be returned)246 //247 typedef struct llama_batch {248 int32_t n_tokens;249 250 llama_token * token;251 float * embd;252 llama_pos * pos;253 int32_t * n_seq_id;254 llama_seq_id ** seq_id;255 int8_t * logits; // TODO: rename this to "output"256 } llama_batch;257 258 enum llama_model_kv_override_type {259 LLAMA_KV_OVERRIDE_TYPE_INT,260 LLAMA_KV_OVERRIDE_TYPE_FLOAT,261 LLAMA_KV_OVERRIDE_TYPE_BOOL,262 LLAMA_KV_OVERRIDE_TYPE_STR,263 };264 265 struct llama_model_kv_override {266 enum llama_model_kv_override_type tag;267 268 char key[128];269 270 union {271 int64_t val_i64;272 double val_f64;273 bool val_bool;274 char val_str[128];275 };276 };277 278 struct llama_model_params {279 // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)280 ggml_backend_dev_t * devices;281 282 int32_t n_gpu_layers; // number of layers to store in VRAM283 enum llama_split_mode split_mode; // how to split the model across multiple GPUs284 285 // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE286 int32_t main_gpu;287 288 // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()289 const float * tensor_split;290 291 // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.292 // If the provided progress_callback returns true, model loading continues.293 // If it returns false, model loading is immediately aborted.294 llama_progress_callback progress_callback;295 296 // context pointer passed to the progress callback297 void * progress_callback_user_data;298 299 // override key-value pairs of the model meta data300 const struct llama_model_kv_override * kv_overrides;301 302 // Keep the booleans together to avoid misalignment during copy-by-value.303 bool vocab_only; // only load the vocabulary, no weights304 bool use_mmap; // use mmap if possible305 bool use_mlock; // force system to keep model in RAM306 bool check_tensors; // validate model tensor data307 };308 309 // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations310 // https://github.com/ggerganov/llama.cpp/pull/7544311 struct llama_context_params {312 uint32_t n_ctx; // text context, 0 = from model313 uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode314 uint32_t n_ubatch; // physical maximum batch size315 uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)316 int32_t n_threads; // number of threads to use for generation317 int32_t n_threads_batch; // number of threads to use for batch processing318 319 enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`320 enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id321 enum llama_attention_type attention_type; // attention type to use for embeddings322 323 // ref: https://github.com/ggerganov/llama.cpp/pull/2054324 float rope_freq_base; // RoPE base frequency, 0 = from model325 float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model326 float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model327 float yarn_attn_factor; // YaRN magnitude scaling factor328 float yarn_beta_fast; // YaRN low correction dim329 float yarn_beta_slow; // YaRN high correction dim330 uint32_t yarn_orig_ctx; // YaRN original context size331 float defrag_thold; // defragment the KV cache if holes/size > thold, < 0 disabled (default)332 333 ggml_backend_sched_eval_callback cb_eval;334 void * cb_eval_user_data;335 336 enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]337 enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]338 339 // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.340 // TODO: move at the end of the struct341 bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)342 bool embeddings; // if true, extract embeddings (together with logits)343 bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU344 bool flash_attn; // whether to use flash attention [EXPERIMENTAL]345 bool no_perf; // whether to measure performance timings346 347 // Abort callback348 // if it returns true, execution of llama_decode() will be aborted349 // currently works only with CPU execution350 ggml_abort_callback abort_callback;351 void * abort_callback_data;352 };353 354 // model quantization parameters355 typedef struct llama_model_quantize_params {356 int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()357 enum llama_ftype ftype; // quantize to this llama_ftype358 enum ggml_type output_tensor_type; // output tensor type359 enum ggml_type token_embedding_type; // token embeddings tensor type360 bool allow_requantize; // allow quantizing non-f32/f16 tensors361 bool quantize_output_tensor; // quantize output.weight362 bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored363 bool pure; // quantize all tensors to the default type364 bool keep_split; // quantize to the same number of shards365 void * imatrix; // pointer to importance matrix data366 void * kv_overrides; // pointer to vector containing overrides367 } llama_model_quantize_params;368 369 typedef struct llama_logit_bias {370 llama_token token;371 float bias;372 } llama_logit_bias;373 374 typedef struct llama_sampler_chain_params {375 bool no_perf; // whether to measure performance timings376 } llama_sampler_chain_params;377 378 // used in chat template379 typedef struct llama_chat_message {380 const char * role;381 const char * content;382 } llama_chat_message;383 384 // lora adapter385 struct llama_adapter_lora;386 387 // Helpers for getting default parameters388 // TODO: update API to start accepting pointers to params structs (https://github.com/ggerganov/llama.cpp/discussions/9172)389 LLAMA_API struct llama_model_params llama_model_default_params(void);390 LLAMA_API struct llama_context_params llama_context_default_params(void);391 LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void);392 LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);393 394 // Initialize the llama + ggml backend395 // If numa is true, use NUMA optimizations396 // Call once at the start of the program397 LLAMA_API void llama_backend_init(void);398 399 // Call once at the end of the program - currently only used for MPI400 LLAMA_API void llama_backend_free(void);401 402 //optional:403 LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);404 405 // Optional: an auto threadpool gets created in ggml if not passed explicitly406 LLAMA_API void llama_attach_threadpool(407 struct llama_context * ctx,408 ggml_threadpool_t threadpool,409 ggml_threadpool_t threadpool_batch);410 411 LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);412 413 DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(414 const char * path_model,415 struct llama_model_params params),416 "use llama_model_load_from_file instead");417 418 // Load the model from a file419 // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf420 // If the split file name does not follow this pattern, use llama_model_load_from_splits421 LLAMA_API struct llama_model * llama_model_load_from_file(422 const char * path_model,423 struct llama_model_params params);424 425 // Load the model from multiple splits (support custom naming scheme)426 // The paths must be in the correct order427 LLAMA_API struct llama_model * llama_model_load_from_splits(428 const char ** paths,429 size_t n_paths,430 struct llama_model_params params);431 432 DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),433 "use llama_model_free instead");434 435 LLAMA_API void llama_model_free(struct llama_model * model);436 437 LLAMA_API struct llama_context * llama_init_from_model(438 struct llama_model * model,439 struct llama_context_params params);440 441 DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(442 struct llama_model * model,443 struct llama_context_params params),444 "use llama_init_from_model instead");445 446 // Frees all allocated memory447 LLAMA_API void llama_free(struct llama_context * ctx);448 449 LLAMA_API int64_t llama_time_us(void);450 451 LLAMA_API size_t llama_max_devices(void);452 453 LLAMA_API bool llama_supports_mmap (void);454 LLAMA_API bool llama_supports_mlock (void);455 LLAMA_API bool llama_supports_gpu_offload(void);456 LLAMA_API bool llama_supports_rpc (void);457 458 LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);459 LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);460 LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);461 LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);462 463 DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");464 DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");465 DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead");466 DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead");467 468 DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");469 470 LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);471 LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx);472 473 LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);474 LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model);475 476 LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);477 LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model);478 LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model);479 LLAMA_API int32_t llama_model_n_head (const struct llama_model * model);480 481 // Get the model's RoPE frequency scaling factor482 LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);483 484 LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab);485 486 LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);487 488 // Functions to access the model's GGUF metadata scalar values489 // - The functions return the length of the string on success, or -1 on failure490 // - The output string is always null-terminated and cleared on failure491 // - When retrieving a string, an extra byte must be allocated to account for the null terminator492 // - GGUF array values are not supported by these functions493 494 // Get metadata value as a string by key name495 LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);496 497 // Get the number of metadata key/value pairs498 LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);499 500 // Get metadata key name by index501 LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);502 503 // Get metadata value as a string by index504 LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);505 506 // Get a string describing the model type507 LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);508 509 // Returns the total size of all the tensors in the model in bytes510 LLAMA_API uint64_t llama_model_size(const struct llama_model * model);511 512 // Get the default chat template. Returns nullptr if not available513 // If name is NULL, returns the default chat template514 LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);515 516 // Returns the total number of parameters in the model517 LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);518 519 // Returns true if the model contains an encoder that requires llama_encode() call520 LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);521 522 // Returns true if the model contains a decoder that requires llama_decode() call523 LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);524 525 // For encoder-decoder models, this function returns id of the token that must be provided526 // to the decoder to start generating output sequence. For other models, it returns -1.527 LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);528 529 // Returns true if the model is recurrent (like Mamba, RWKV, etc.)530 LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);531 532 // Returns 0 on success533 LLAMA_API uint32_t llama_model_quantize(534 const char * fname_inp,535 const char * fname_out,536 const llama_model_quantize_params * params);537 538 //539 // Adapters540 //541 542 // Load a LoRA adapter from file543 LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(544 struct llama_model * model,545 const char * path_lora);546 547 // Manually free a LoRA adapter548 // Note: loaded adapters will be free when the associated model is deleted549 LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);550 551 // The following functions operate on a llama_context, hence the naming: llama_verb_...552 553 // Add a loaded LoRA adapter to given context554 // This will not modify model's weight555 LLAMA_API int32_t llama_set_adapter_lora(556 struct llama_context * ctx,557 struct llama_adapter_lora * adapter,558 float scale);559 560 // Remove a specific LoRA adapter from given context561 // Return -1 if the adapter is not present in the context562 LLAMA_API int32_t llama_rm_adapter_lora(563 struct llama_context * ctx,564 struct llama_adapter_lora * adapter);565 566 // Remove all LoRA adapters from given context567 LLAMA_API void llama_clear_adapter_lora(struct llama_context * ctx);568 569 // Apply a loaded control vector to a llama_context, or if data is NULL, clear570 // the currently loaded vector.571 // n_embd should be the size of a single layer's control, and data should point572 // to an n_embd x n_layers buffer starting from layer 1.573 // il_start and il_end are the layer range the vector should apply to (both inclusive)574 // See llama_control_vector_load in common to load a control vector.575 LLAMA_API int32_t llama_apply_adapter_cvec(576 struct llama_context * ctx,577 const float * data,578 size_t len,579 int32_t n_embd,580 int32_t il_start,581 int32_t il_end);582 583 //584 // KV cache585 //586 587 // TODO: remove llama_kv_cache_view_* API588 589 // Information associated with an individual cell in the KV cache view.590 struct llama_kv_cache_view_cell {591 // The position for this cell. Takes KV cache shifts into account.592 // May be negative if the cell is not populated.593 llama_pos pos;594 };595 596 // An updateable view of the KV cache.597 struct llama_kv_cache_view {598 // Number of KV cache cells. This will be the same as the context size.599 int32_t n_cells;600 601 // Maximum number of sequences that can exist in a cell. It's not an error602 // if there are more sequences in a cell than this value, however they will603 // not be visible in the view cells_sequences.604 int32_t n_seq_max;605 606 // Number of tokens in the cache. For example, if there are two populated607 // cells, the first with 1 sequence id in it and the second with 2 sequence608 // ids then you'll have 3 tokens.609 int32_t token_count;610 611 // Number of populated cache cells.612 int32_t used_cells;613 614 // Maximum contiguous empty slots in the cache.615 int32_t max_contiguous;616 617 // Index to the start of the max_contiguous slot range. Can be negative618 // when cache is full.619 int32_t max_contiguous_idx;620 621 // Information for an individual cell.622 struct llama_kv_cache_view_cell * cells;623 624 // The sequences for each cell. There will be n_seq_max items per cell.625 llama_seq_id * cells_sequences;626 };627 628 // Create an empty KV cache view. (use only for debugging purposes)629 LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max);630 631 // Free a KV cache view. (use only for debugging purposes)632 LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view);633 634 // Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)635 // TODO: change signature to llama_kv_cache_view_update(struct llama_kv_cache_view * view, const struct llama_context * ctx)636 LLAMA_API void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view);637 638 ///639 640 // Returns the number of tokens in the KV cache (slow, use only for debug)641 // If a KV cell has multiple sequences assigned to it, it will be counted multiple times642 LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx);643 644 // Returns the number of used KV cells (i.e. have at least one sequence assigned to them)645 LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx);646 647 // Clear the KV cache - both cell info is erased and KV data is zeroed648 LLAMA_API void llama_kv_cache_clear(649 struct llama_context * ctx);650 651 // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)652 // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails653 // seq_id < 0 : match any sequence654 // p0 < 0 : [0, p1]655 // p1 < 0 : [p0, inf)656 LLAMA_API bool llama_kv_cache_seq_rm(657 struct llama_context * ctx,658 llama_seq_id seq_id,659 llama_pos p0,660 llama_pos p1);661 662 // Copy all tokens that belong to the specified sequence to another sequence663 // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence664 // p0 < 0 : [0, p1]665 // p1 < 0 : [p0, inf)666 LLAMA_API void llama_kv_cache_seq_cp(667 struct llama_context * ctx,668 llama_seq_id seq_id_src,669 llama_seq_id seq_id_dst,670 llama_pos p0,671 llama_pos p1);672 673 // Removes all tokens that do not belong to the specified sequence674 LLAMA_API void llama_kv_cache_seq_keep(675 struct llama_context * ctx,676 llama_seq_id seq_id);677 678 // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)679 // If the KV cache is RoPEd, the KV data is updated accordingly:680 // - lazily on next llama_decode()681 // - explicitly with llama_kv_cache_update()682 // p0 < 0 : [0, p1]683 // p1 < 0 : [p0, inf)684 LLAMA_API void llama_kv_cache_seq_add(685 struct llama_context * ctx,686 llama_seq_id seq_id,687 llama_pos p0,688 llama_pos p1,689 llama_pos delta);690 691 // Integer division of the positions by factor of `d > 1`692 // If the KV cache is RoPEd, the KV data is updated accordingly:693 // - lazily on next llama_decode()694 // - explicitly with llama_kv_cache_update()695 // p0 < 0 : [0, p1]696 // p1 < 0 : [p0, inf)697 LLAMA_API void llama_kv_cache_seq_div(698 struct llama_context * ctx,699 llama_seq_id seq_id,700 llama_pos p0,701 llama_pos p1,702 int d);703 704 // Returns the largest position present in the KV cache for the specified sequence705 LLAMA_API llama_pos llama_kv_cache_seq_pos_max(706 struct llama_context * ctx,707 llama_seq_id seq_id);708 709 // TODO: the llama_kv_cache_defrag and llama_kv_cache_update API tightly couples llama_context with llama_kv_cache710 // how to avoid this?711 712 // Defragment the KV cache713 // This will be applied:714 // - lazily on next llama_decode()715 // - explicitly with llama_kv_cache_update()716 LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx);717 718 // Apply the KV cache updates (such as K-shifts, defragmentation, etc.)719 LLAMA_API void llama_kv_cache_update(struct llama_context * ctx);720 721 // Check if the context supports KV cache shifting722 LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx);723 724 //725 // State / sessions726 //727 728 // Returns the *actual* size in bytes of the state729 // (logits, embedding and kv_cache)730 // Only use when saving the state, not when restoring it, otherwise the size may be too small.731 LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);732 LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),733 "use llama_state_get_size instead");734 735 // Copies the state to the specified destination address.736 // Destination needs to have allocated enough memory.737 // Returns the number of bytes copied738 LLAMA_API size_t llama_state_get_data(739 struct llama_context * ctx,740 uint8_t * dst,741 size_t size);742 LLAMA_API DEPRECATED(size_t llama_copy_state_data(743 struct llama_context * ctx,744 uint8_t * dst),745 "use llama_state_get_data instead");746 747 // Set the state reading from the specified address748 // Returns the number of bytes read749 LLAMA_API size_t llama_state_set_data(750 struct llama_context * ctx,751 const uint8_t * src,752 size_t size);753 LLAMA_API DEPRECATED(size_t llama_set_state_data(754 struct llama_context * ctx,755 const uint8_t * src),756 "use llama_state_set_data instead");757 758 // Save/load session file759 LLAMA_API bool llama_state_load_file(760 struct llama_context * ctx,761 const char * path_session,762 llama_token * tokens_out,763 size_t n_token_capacity,764 size_t * n_token_count_out);765 LLAMA_API DEPRECATED(bool llama_load_session_file(766 struct llama_context * ctx,767 const char * path_session,768 llama_token * tokens_out,769 size_t n_token_capacity,770 size_t * n_token_count_out),771 "use llama_state_load_file instead");772 773 LLAMA_API bool llama_state_save_file(774 struct llama_context * ctx,775 const char * path_session,776 const llama_token * tokens,777 size_t n_token_count);778 LLAMA_API DEPRECATED(bool llama_save_session_file(779 struct llama_context * ctx,780 const char * path_session,781 const llama_token * tokens,782 size_t n_token_count),783 "use llama_state_save_file instead");784 785 // Get the exact size needed to copy the KV cache of a single sequence786 LLAMA_API size_t llama_state_seq_get_size(787 struct llama_context * ctx,788 llama_seq_id seq_id);789 790 // Copy the KV cache of a single sequence into the specified buffer791 LLAMA_API size_t llama_state_seq_get_data(792 struct llama_context * ctx,793 uint8_t * dst,794 size_t size,795 llama_seq_id seq_id);796 797 // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence798 // Returns:799 // - Positive: Ok800 // - Zero: Failed to load801 LLAMA_API size_t llama_state_seq_set_data(802 struct llama_context * ctx,803 const uint8_t * src,804 size_t size,805 llama_seq_id dest_seq_id);806 807 LLAMA_API size_t llama_state_seq_save_file(808 struct llama_context * ctx,809 const char * filepath,810 llama_seq_id seq_id,811 const llama_token * tokens,812 size_t n_token_count);813 814 LLAMA_API size_t llama_state_seq_load_file(815 struct llama_context * ctx,816 const char * filepath,817 llama_seq_id dest_seq_id,818 llama_token * tokens_out,819 size_t n_token_capacity,820 size_t * n_token_count_out);821 822 //823 // Decoding824 //825 826 // Return batch for single sequence of tokens827 // The sequence ID will be fixed to 0828 // The position of the tokens will be tracked automatically by llama_decode829 //830 // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it831 //832 LLAMA_API struct llama_batch llama_batch_get_one(833 llama_token * tokens,834 int32_t n_tokens);835 836 // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens837 // Each token can be assigned up to n_seq_max sequence ids838 // The batch has to be freed with llama_batch_free()839 // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)840 // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token841 // The rest of the llama_batch members are allocated with size n_tokens842 // All members are left uninitialized843 LLAMA_API struct llama_batch llama_batch_init(844 int32_t n_tokens,845 int32_t embd,846 int32_t n_seq_max);847 848 // Frees a batch of tokens allocated with llama_batch_init()849 LLAMA_API void llama_batch_free(struct llama_batch batch);850 851 // Processes a batch of tokens with the ecoder part of the encoder-decoder model.852 // Stores the encoder output internally for later use by the decoder cross-attention layers.853 // 0 - success854 // < 0 - error. the KV cache state is restored to the state before this call855 LLAMA_API int32_t llama_encode(856 struct llama_context * ctx,857 struct llama_batch batch);858 859 // Positive return values does not mean a fatal error, but rather a warning.860 // 0 - success861 // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)862 // < 0 - error. the KV cache state is restored to the state before this call863 LLAMA_API int32_t llama_decode(864 struct llama_context * ctx,865 struct llama_batch batch);866 867 // Set the number of threads used for decoding868 // n_threads is the number of threads used for generation (single token)869 // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)870 LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);871 872 // Get the number of threads used for generation of a single token.873 LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);874 875 // Get the number of threads used for prompt and batch processing (multiple token).876 LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);877 878 // Set whether the model is in embeddings mode or not879 // If true, embeddings will be returned but logits will not880 LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);881 882 // Set whether to use causal attention or not883 // If set to true, the model will only attend to the past tokens884 LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);885 886 // Set abort callback887 LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);888 889 // Wait until all computations are finished890 // This is automatically done when using one of the functions below to obtain the computation results891 // and is not necessary to call it explicitly in most cases892 LLAMA_API void llama_synchronize(struct llama_context * ctx);893 894 // Token logits obtained from the last call to llama_decode()895 // The logits for which llama_batch.logits[i] != 0 are stored contiguously896 // in the order they have appeared in the batch.897 // Rows: number of tokens for which llama_batch.logits[i] != 0898 // Cols: n_vocab899 LLAMA_API float * llama_get_logits(struct llama_context * ctx);900 901 // Logits for the ith token. For positive indices, Equivalent to:902 // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab903 // Negative indicies can be used to access logits in reverse order, -1 is the last logit.904 // returns NULL for invalid ids.905 LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);906 907 // Get all output token embeddings.908 // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,909 // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously910 // in the order they have appeared in the batch.911 // shape: [n_outputs*n_embd]912 // Otherwise, returns NULL.913 LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);914 915 // Get the embeddings for the ith token. For positive indices, Equivalent to:916 // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd917 // Negative indicies can be used to access embeddings in reverse order, -1 is the last embedding.918 // shape: [n_embd] (1-dimensional)919 // returns NULL for invalid ids.920 LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);921 922 // Get the embeddings for a sequence id923 // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE924 // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[1] with the rank of the sequence925 // otherwise: float[n_embd] (1-dimensional)926 LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);927 928 //929 // Vocab930 //931 932 LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);933 934 LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);935 936 LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);937 938 // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)939 LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);940 941 // Identify if Token Id is a control token or a render-able token942 LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);943 944 // Special tokens945 LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence946 LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence947 LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn948 LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator949 LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line950 LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding951 952 LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);953 LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);954 955 LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);956 LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);957 LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);958 LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);959 LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);960 LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);961 962 DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");963 DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");964 DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");965 DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");966 DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");967 DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");968 DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");969 DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");970 DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");971 DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");972 DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");973 DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");974 DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");975 DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");976 DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");977 DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");978 DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");979 DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");980 DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");981 DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");982 983 // CLS is equivalent to BOS984 DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification985 "use llama_vocab_bos instead");986 987 //988 // Tokenization989 //990 // The API is thread-safe.991 //992 993 /// @details Convert the provided text into tokens.994 /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.995 /// @return Returns the number of tokens on success, no more than n_tokens_max996 /// @return Returns a negative number on failure - the number of tokens that would have been returned997 /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.998 /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated999 /// as plaintext. Does not insert a leading space.1000 LLAMA_API int32_t llama_tokenize(1001 const struct llama_vocab * vocab,1002 const char * text,1003 int32_t text_len,1004 llama_token * tokens,1005 int32_t n_tokens_max,1006 bool add_special,1007 bool parse_special);1008 1009 // Token Id -> Piece.1010 // Uses the vocabulary in the provided context.1011 // Does not write null terminator to the buffer.1012 // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')1013 // @param special If true, special tokens are rendered in the output.1014 LLAMA_API int32_t llama_token_to_piece(1015 const struct llama_vocab * vocab,1016 llama_token token,1017 char * buf,1018 int32_t length,1019 int32_t lstrip,1020 bool special);1021 1022 /// @details Convert the provided tokens into text (inverse of llama_tokenize()).1023 /// @param text The char pointer must be large enough to hold the resulting text.1024 /// @return Returns the number of chars/bytes on success, no more than text_len_max.1025 /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.1026 /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.1027 /// @param unparse_special If true, special tokens are rendered in the output.1028 LLAMA_API int32_t llama_detokenize(1029 const struct llama_vocab * vocab,1030 const llama_token * tokens,1031 int32_t n_tokens,1032 char * text,1033 int32_t text_len_max,1034 bool remove_special,1035 bool unparse_special);1036 1037 //1038 // Chat templates1039 //1040 1041 /// Apply chat template. Inspired by hf apply_chat_template() on python.1042 /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model"1043 /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template1044 /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model’s default chat template will be used instead.1045 /// @param chat Pointer to a list of multiple llama_chat_message1046 /// @param n_msg Number of llama_chat_message in this chat1047 /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.1048 /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)1049 /// @param length The size of the allocated buffer1050 /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.1051 LLAMA_API int32_t llama_chat_apply_template(1052 const char * tmpl,1053 const struct llama_chat_message * chat,1054 size_t n_msg,1055 bool add_ass,1056 char * buf,1057 int32_t length);1058 1059 // Get list of built-in chat templates1060 LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);1061 1062 //1063 // Sampling API1064 //1065 // Sample usage:1066 //1067 // // prepare the sampling chain at the start1068 // auto sparams = llama_sampler_chain_default_params();1069 //1070 // llama_sampler * smpl = llama_sampler_chain_init(sparams);1071 //1072 // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50));1073 // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1));1074 // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8));1075 //1076 // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat"1077 // // this sampler will be responsible to select the actual token1078 // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed));1079 //1080 // ...1081 //1082 // // decoding loop:1083 // while (...) {1084 // ...1085 //1086 // llama_decode(ctx, batch);1087 //1088 // // sample from the logits of the last token in the batch1089 // const llama_token id = llama_sampler_sample(smpl, ctx, -1);1090 //1091 // // accepting the token updates the internal state of certain samplers (e.g. grammar, repetition, etc.)1092 // llama_sampler_accept(smpl, id);1093 // ...1094 // }1095 //1096 // llama_sampler_free(smpl);1097 //1098 // TODO: In the future, llama_sampler will be utilized to offload the sampling to the backends (e.g. GPU).1099 //1100 1101 typedef void * llama_sampler_context_t;1102 1103 // user code can implement the interface below in order to create custom llama_sampler1104 struct llama_sampler_i {1105 const char * (*name) (const struct llama_sampler * smpl); // can be NULL1106 void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL1107 void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required1108 void (*reset) ( struct llama_sampler * smpl); // can be NULL1109 struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL1110 void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL1111 1112 // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph1113 //void (*apply_ggml) (struct llama_sampler * smpl, ...);1114 };1115 1116 struct llama_sampler {1117 const struct llama_sampler_i * iface;1118 llama_sampler_context_t ctx;1119 };1120 1121 // mirror of llama_sampler_i:1122 LLAMA_API struct llama_sampler * llama_sampler_init (const struct llama_sampler_i * iface, llama_sampler_context_t ctx);1123 LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl);1124 LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token);1125 LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p);1126 LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl);1127 LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl);1128 // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)1129 LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl);1130 1131 // llama_sampler_chain1132 // a type of llama_sampler that can chain multiple samplers one after another1133 1134 LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params);1135 1136 // important: takes ownership of the sampler object and will free it when llama_sampler_free is called1137 LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl);1138 LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i);1139 LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain);1140 1141 // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed1142 LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i);1143 1144 // available samplers:1145 1146 LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);1147 LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed);1148 1149 /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.1150 /// NOTE: Avoid using on the full vocabulary as the sorting can become slow. For example, apply top-k or top-p sampling first.1151 DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void),1152 "will be removed in the future (see https://github.com/ggerganov/llama.cpp/pull/9896#discussion_r1800920915)");1153 1154 /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.097511155 LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k);1156 1157 /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.097511158 LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep);1159 1160 /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/38411161 LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep);1162 1163 /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.1164 LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep);1165 1166 /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf1167 LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t);1168 1169 /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.1170 LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent);1171 1172 /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/63351173 LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed);1174 1175 /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.1176 /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.1177 /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.1178 /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.1179 /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.1180 /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.1181 LLAMA_API struct llama_sampler * llama_sampler_init_mirostat(1182 int32_t n_vocab,1183 uint32_t seed,1184 float tau,1185 float eta,1186 int32_t m);1187 1188 /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.1189 /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.1190 /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.1191 /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.1192 /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.1193 LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2(1194 uint32_t seed,1195 float tau,1196 float eta);1197 1198 LLAMA_API struct llama_sampler * llama_sampler_init_grammar(1199 const struct llama_vocab * vocab,1200 const char * grammar_str,