Brunobkr/llama.cpp_AlgMor24_github
ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.
03.1k
1#include "llama-context.h"2 3#include "ggml.h"4#include "llama-arch.h"5#include "llama-graph.h"6#include "llama-impl.h"7#include "llama-batch.h"8#include "llama-io.h"9#include "llama-memory.h"10#include "llama-mmap.h"11#include "llama-model.h"12#include "llama-ext.h"13#include "llama-sampler.h"14#include "llama.h"15 16#include <cinttypes>17#include <cmath>18#include <cstring>19#include <limits>20#include <stdexcept>21#include <string>22 23//24// llama_context25//26 27static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {28 switch (ctx_type) {29 case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT;30 case LLAMA_CONTEXT_TYPE_MTP : return LLM_GRAPH_TYPE_DECODER_MTP;31 }32 throw std::runtime_error("Unsupported ctx type");33}34 35struct llm_fused_op_probe {36 llm_fused_op op;37 const char * name;38 uint32_t n_tokens_per_seq;39};40 41static const llm_fused_op_probe llm_fused_op_flash_attn_probe = {42 /*.op =*/ LLM_FUSED_OP_FLASH_ATTN,43 /*.name =*/ "Flash Attention",44 /*.n_tokens_per_seq =*/ 1,45};46 47static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = {48 /*.op =*/ LLM_FUSED_OP_GDN_AR,49 /*.name =*/ "fused Gated Delta Net (autoregressive)",50 /*.n_tokens_per_seq =*/ 1,51};52 53static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = {54 /*.op =*/ LLM_FUSED_OP_GDN_CH,55 /*.name =*/ "fused Gated Delta Net (chunked)",56 /*.n_tokens_per_seq =*/ 16,57};58 59static const llm_fused_op_probe llm_fused_op_lid_probe = {60 /*.op =*/ LLM_FUSED_OP_LIGHTNING_INDEXER,61 /*.name =*/ "Lightning Indexer",62 /*.n_tokens_per_seq =*/ 1,63};64 65static const llm_fused_op_probe llm_fused_op_dsv4_hc_pre_probe = {66 /*.op =*/ LLM_FUSED_OP_DSV4_HC_PRE,67 /*.name =*/ "fused DeepSeek V4 HC pre",68 /*.n_tokens_per_seq =*/ 1,69};70 71static const llm_fused_op_probe llm_fused_op_dsv4_hc_comb_probe = {72 /*.op =*/ LLM_FUSED_OP_DSV4_HC_COMB,73 /*.name =*/ "fused DeepSeek V4 HC comb",74 /*.n_tokens_per_seq =*/ 1,75};76 77static const llm_fused_op_probe llm_fused_op_dsv4_hc_post_probe = {78 /*.op =*/ LLM_FUSED_OP_DSV4_HC_POST,79 /*.name =*/ "fused DeepSeek V4 HC post",80 /*.n_tokens_per_seq =*/ 1,81};82 83llama_context::llama_context(84 const llama_model & model,85 llama_context_params params) :86 model(model),87 cvec(std::make_unique<llama_adapter_cvec>()),88 loras(std::make_unique<llama_adapter_loras>()),89 balloc(std::make_unique<llama_batch_allocr>(model.hparams.n_pos_per_embd())) {90 // TODO warning when creating llama_context with awkward ctx size that is not a power of 2,91 // may need to be backend-dependent92 LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__);93 94 t_start_us = model.t_start_us;95 t_load_us = model.t_load_us;96 97 const auto & hparams = model.hparams;98 99 cparams.n_seq_max = std::max(1u, params.n_seq_max);100 if (cparams.n_seq_max > LLAMA_MAX_SEQ) {101 throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ));102 }103 104 cparams.n_rs_seq = params.n_rs_seq;105 if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) {106 LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n",107 __func__, cparams.n_rs_seq);108 cparams.n_rs_seq = 0;109 }110 111 cparams.n_threads = params.n_threads;112 cparams.n_threads_batch = params.n_threads_batch;113 cparams.yarn_ext_factor = params.yarn_ext_factor >= 0.0f ? params.yarn_ext_factor : hparams.yarn_ext_factor;114 cparams.yarn_attn_factor = params.yarn_attn_factor >= 0.0f ? params.yarn_attn_factor : hparams.yarn_attn_factor;115 cparams.yarn_beta_fast = params.yarn_beta_fast >= 0.0f ? params.yarn_beta_fast : hparams.yarn_beta_fast;116 cparams.yarn_beta_slow = params.yarn_beta_slow >= 0.0f ? params.yarn_beta_slow : hparams.yarn_beta_slow;117 cparams.embeddings = params.embeddings;118 cparams.embeddings_nextn = false;119 cparams.embeddings_nextn_masked = false;120 cparams.offload_kqv = params.offload_kqv;121 cparams.no_perf = params.no_perf;122 cparams.warmup = false;123 124 // +1: id n_layer() taps the output of the last layer ("input" of the head)125 cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);126 embd_layer_inp.resize(hparams.n_layer() + 1);127 128 cparams.ctx_type = params.ctx_type;129 cparams.pooling_type = params.pooling_type;130 131 cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;132 cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;133 cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;134 135 cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :136 hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :137 hparams.n_ctx_train;138 139 cparams.cb_eval = params.cb_eval;140 cparams.cb_eval_user_data = params.cb_eval_user_data;141 142 cparams.ctx_other = nullptr;143 144 // TODO: more generic145 if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) {146 if (params.ctx_other == nullptr) {147 // TODO: change from runtime_error to llama_exception to avoid printing error message148 throw std::runtime_error("Gemma4Assistant requires ctx_other to be set (this warning is normal during memory fitting)");149 }150 151 cparams.ctx_other = params.ctx_other;152 }153 154 if (model.arch == LLM_ARCH_EAGLE3 || model.arch == LLM_ARCH_DFLASH) {155 if (model.tok_embd == nullptr || model.output == nullptr) {156 if (params.ctx_other == nullptr) {157 throw std::runtime_error(model.arch_name() + " requires ctx_other to be set (this warning is normal during memory fitting)");158 }159 cparams.ctx_other = params.ctx_other;160 }161 }162 163 auto rope_scaling_type = params.rope_scaling_type;164 if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {165 rope_scaling_type = hparams.rope_scaling_type_train;166 }167 168 if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {169 cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none170 }171 172 if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'173 cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;174 }175 176 if (cparams.yarn_ext_factor != 0) {177 static auto get_mscale = [](float scale, float mscale) {178 return scale <= 1.0f ? 1.0f : (0.1f * mscale * logf(scale) + 1.0f);179 };180 181 const float factor = 1.0f / cparams.rope_freq_scale;182 183 // ref: https://github.com/huggingface/transformers/blob/6d00f6b0a5679c36510f203e4226e36f517c3032/src/transformers/modeling_rope_utils.py#L336-L348184 if (hparams.rope_yarn_log_mul != 0.0f) {185 // note: here we assume `mscale == 1.0f`186 // TODO: start reading the actual value of mscale and handle the case where it is not 1.0f187 float mscale = 1.0f;188 const float mscale_all_dims = hparams.rope_yarn_log_mul;189 190 // [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]191 // special-case DEEPSEEK v2:192 // https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/main/config.json#L42-L43193 if (model.arch == LLM_ARCH_DEEPSEEK2 && mscale_all_dims != 1.0f) {194 mscale = mscale_all_dims;195 }196 197 cparams.yarn_attn_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dims);198 199 LLAMA_LOG_WARN("%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n",200 __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims);201 } else {202 cparams.yarn_attn_factor = get_mscale(factor, 1.0f);203 }204 205 // when YARN is applied with yarn_ext_factor != 0.0f, we need to cancel this factor:206 // https://github.com/ggml-org/llama.cpp/blob/a81a569577cc38b32558958b048228150be63eae/ggml/src/ggml-cpu/ops.cpp#L5541-L5544207 //208 // ref: https://github.com/ggml-org/llama.cpp/discussions/7416209 // https://github.com/ggml-org/llama.cpp/pull/17945210 cparams.yarn_attn_factor *= 1.0f / (1.0f + 0.1f * logf(factor));211 }212 213 cparams.yarn_attn_factor *= hparams.rope_attn_factor;214 215 if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {216 if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {217 cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;218 } else {219 cparams.pooling_type = hparams.pooling_type;220 }221 }222 223 if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {224 cparams.causal_attn = hparams.causal_attn;225 } else {226 cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;227 }228 229 cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED;230 cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO;231 232 cparams.fused_gdn_ar = true;233 cparams.fused_gdn_ch = true;234 cparams.auto_fgdn = true;235 236 cparams.fused_lid = true;237 cparams.auto_flid = true;238 239 cparams.fused_dsv4_hc_pre = true;240 cparams.fused_dsv4_hc_comb = true;241 cparams.fused_dsv4_hc_post = true;242 cparams.auto_fhc = true;243 244 // with causal attention, the batch size is limited by the context size245 cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;246 247 cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);248 249 cparams.n_outputs_max = params.n_outputs_max == 0 || llama_model_has_encoder(&model) ? cparams.n_batch : params.n_outputs_max;250 cparams.n_outputs_max_per_seq = params.n_outputs_max_per_seq == 0 ?251 cparams.n_outputs_max : std::min(params.n_outputs_max_per_seq, cparams.n_outputs_max);252 253 // Initialize backend samplers here so they are part of the sampling graph254 // before the reserve passes run later in this function. This avoids a later255 // re-reserve when graph nodes change.256 if (params.samplers != nullptr && params.n_samplers > 0) {257 for (size_t i = 0; i < params.n_samplers; ++i) {258 const auto & config = params.samplers[i];259 260 if (llama_sampler_chain_get(config.sampler, -1) == nullptr) {261 throw std::runtime_error("the backend samplers must be of type llama_sampler_chain");262 }263 264 if (set_sampler(config.seq_id, config.sampler)) {265 const int n_samplers = llama_sampler_chain_n(config.sampler);266 267 LLAMA_LOG_INFO("%s: setting backend sampler for seq_id %d (n = %d)\n", __func__, config.seq_id, n_samplers);268 }269 }270 }271 272 cparams.op_offload = params.op_offload;273 cparams.kv_unified = params.kv_unified;274 275 // initialized later276 cparams.pipeline_parallel = false;277 278 {279 const char * LLAMA_GRAPH_REUSE_DISABLE = getenv("LLAMA_GRAPH_REUSE_DISABLE");280 graph_reuse_disable = LLAMA_GRAPH_REUSE_DISABLE ? (atoi(LLAMA_GRAPH_REUSE_DISABLE) != 0) : graph_reuse_disable;281 282 if (graph_reuse_disable) {283 LLAMA_LOG_WARN("%s: graph reuse disabled\n", __func__);284 }285 }286 287 // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732288 cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);289 290 if (cparams.kv_unified) {291 cparams.n_ctx_seq = cparams.n_ctx;292 } else {293 cparams.n_ctx_seq = cparams.n_ctx / cparams.n_seq_max;294 cparams.n_ctx_seq = GGML_PAD(cparams.n_ctx_seq, 256);295 296 if (cparams.n_ctx_seq == 0) {297 throw std::runtime_error("n_ctx_seq == 0");298 }299 300 if (cparams.n_ctx != cparams.n_ctx_seq * cparams.n_seq_max) {301 cparams.n_ctx = cparams.n_ctx_seq * cparams.n_seq_max;302 LLAMA_LOG_WARN("%s: n_ctx is not divisible by n_seq_max - rounding down to %u\n", __func__, cparams.n_ctx);303 }304 }305 306 LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max);307 LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);308 LLAMA_LOG_INFO("%s: n_ctx_seq = %u\n", __func__, cparams.n_ctx_seq);309 LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);310 LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);311 LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn);312 LLAMA_LOG_INFO("%s: flash_attn = %s\n", __func__, llama_flash_attn_type_name(params.flash_attn_type));313 LLAMA_LOG_INFO("%s: kv_unified = %s\n", __func__, cparams.kv_unified ? "true" : "false");314 LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);315 LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);316 LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq);317 LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max);318 LLAMA_LOG_INFO("%s: n_outputs_max_per_seq = %u\n", __func__, cparams.n_outputs_max_per_seq);319 320 if (cparams.n_ctx_seq < hparams.n_ctx_train) {321 LLAMA_LOG_INFO("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",322 __func__, cparams.n_ctx_seq, hparams.n_ctx_train);323 }324 325 if (cparams.n_ctx_seq > hparams.n_ctx_train) {326 LLAMA_LOG_WARN("%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n",327 __func__, cparams.n_ctx_seq, hparams.n_ctx_train);328 }329 330 if (!hparams.vocab_only) {331 // GPU backends332 for (const auto & dev : model.devices) {333 ggml_backend_t backend = ggml_backend_dev_init(dev.dev, nullptr);334 if (backend == nullptr) {335 throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev.dev)));336 }337 backends.emplace_back(backend);338 }339 340 // add ACCEL backends (such as BLAS)341 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {342 ggml_backend_dev_t dev = ggml_backend_dev_get(i);343 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {344 ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);345 if (backend == nullptr) {346 throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));347 }348 backends.emplace_back(backend);349 }350 }351 352 // add CPU backend353 backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);354 if (backend_cpu == nullptr) {355 throw std::runtime_error("failed to initialize CPU backend");356 }357 backends.emplace_back(backend_cpu);358 359 // create a list of the set_n_threads functions in the backends360 for (auto & backend : backends) {361 ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());362 ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;363 if (reg) {364 auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");365 if (ggml_backend_set_n_threads_fn) {366 set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn);367 }368 }369 }370 371 llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data);372 373 // graph outputs buffer374 {375 if (output_reserve(params.n_seq_max) < params.n_seq_max) {376 throw std::runtime_error("failed to reserve initial output buffer");377 }378 379 LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,380 ggml_backend_buffer_name (buf_output.get()),381 ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0);382 }383 }384 385 // init the memory module386 if (!hparams.vocab_only) {387 llama_memory_params params_mem = {388 /*.type_k =*/ params.type_k,389 /*.type_v =*/ params.type_v,390 /*.swa_full =*/ params.swa_full,391 /*.ctx_type =*/ cparams.ctx_type,392 /*.mem_other =*/ llama_get_memory(cparams.ctx_other),393 };394 395 memory.reset(model.create_memory(params_mem, cparams));396 }397 398 // init backends399 if (!hparams.vocab_only) {400 LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__);401 402 backend_buft.clear();403 backend_ptrs.clear();404 backend_buf_exp_size.clear();405 406 for (auto & backend : backends) {407 auto * buft = ggml_backend_get_default_buffer_type(backend.get());408 auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));409 410 if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {411 // use the host buffer of the first device CPU for faster transfer of the intermediate state412 const auto & dev = model.devices[0];413 auto * host_buft = ggml_backend_dev_host_buffer_type(dev.dev);414 if (host_buft) {415 buft = host_buft;416 }417 }418 419 backend_buft.push_back(buft);420 backend_ptrs.push_back(backend.get());421 backend_buf_exp_size.push_back(0);422 }423 424 LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size());425 426 // TODO: move these checks to ggml_backend_sched427 // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary428 bool pipeline_parallel =429 model.n_devices() > 1 &&430 model.n_gpu_layers() > model.hparams.n_layer_all &&431 model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&432 cparams.offload_kqv &&433 !model.has_tensor_overrides();434 435 // pipeline parallelism requires support for async compute and events in all devices436 if (pipeline_parallel) {437 for (auto & backend : backends) {438 auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));439 if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) {440 // ignore CPU backend441 // TODO: should we ignore ACCEL types too?442 continue;443 }444 auto * dev = ggml_backend_get_device(backend.get());445 ggml_backend_dev_props props;446 ggml_backend_dev_get_props(dev, &props);447 if (!props.caps.async || !props.caps.events) {448 // device does not support async compute or events449 pipeline_parallel = false;450 break;451 }452 }453 }454 455 cparams.pipeline_parallel = pipeline_parallel;456 457 if (cparams.pipeline_parallel) {458 LLAMA_LOG_INFO("%s: pipeline parallelism enabled\n", __func__);459 }460 461 sched_reserve();462 463 if (!cparams.flash_attn) {464 if (ggml_is_quantized(params.type_v)) {465 throw std::runtime_error("quantized V cache was requested, but this requires Flash Attention");466 }467 }468 }469 470 // Initialize the full vocabulary token ids for backend samplers.471 {472 const int n_vocab = model.vocab.n_tokens();473 474 sampling.token_ids_full_vocab.resize(n_vocab);475 for (int i = 0; i < n_vocab; ++i) {476 sampling.token_ids_full_vocab[i] = i;477 }478 }479}480 481llama_context::~llama_context() {482 // wait for any pending asynchronous copies into the output buffers before they are freed483 synchronize();484 485 if (!model.hparams.no_alloc) {486 for (size_t i = 0; i < backend_ptrs.size(); ++i) {487 ggml_backend_t backend = backend_ptrs[i];488 ggml_backend_buffer_type_t buft = backend_buft[i];489 490 const size_t size_exp = backend_buf_exp_size[i];491 const size_t size_act = ggml_backend_sched_get_buffer_size(sched.get(), backend);492 if (size_exp == size_act) {493 LLAMA_LOG_DEBUG("%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n",494 __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0));495 } else {496 LLAMA_LOG_WARN("%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n",497 __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0));498 }499 }500 }501 ggml_opt_free(opt_ctx);502}503 504void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs) {505 const char * func = __func__;506 auto resolve = [&](const llm_fused_op_probe & probe, bool & enabled) {507 if (!enabled) {508 return;509 }510 511 const uint32_t n_tokens_probe = probe.n_tokens_per_seq*n_seqs;512 513 auto * gf = graph_reserve(n_tokens_probe, n_seqs, n_tokens_probe, mctx, true);514 if (!gf) {515 throw std::runtime_error(std::string("failed to reserve graph for ") + probe.name + " check");516 }517 518 bool device_mismatch = false;519 for (const auto & node : get_gf_res_reserve()->get_fused_nodes()) {520 if (node.op != probe.op) {521 continue;522 }523 524 GGML_ASSERT(node.il >= 0);525 526 ggml_backend_t backend_fused = ggml_backend_sched_get_tensor_backend(sched.get(), node.tensor);527 ggml_backend_dev_t device_fused = backend_fused ? ggml_backend_get_device(backend_fused) : nullptr;528 529 // TODO: make this descriptor-specific; model.dev_layer() preserves the current behavior,530 // but is still wrong for cases like --no-kv-offload.531 ggml_backend_dev_t device_layer = model.dev_layer(node.il);532 533 if (device_fused != device_layer) {534 LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but %s "535 "is assigned to device %s (usually due to missing support)\n",536 func, node.il,537 device_layer ? ggml_backend_dev_name(device_layer) : "none",538 probe.name,539 device_fused ? ggml_backend_dev_name(device_fused) : "none");540 device_mismatch = true;541 break;542 }543 }544 545 if (device_mismatch) {546 enabled = false;547 LLAMA_LOG_WARN("%s: %s not supported, set to disabled\n", func, probe.name);548 } else {549 enabled = true;550 LLAMA_LOG_INFO("%s: %s enabled\n", func, probe.name);551 }552 };553 554 if (cparams.auto_fa) {555 resolve(llm_fused_op_flash_attn_probe, cparams.flash_attn);556 cparams.auto_fa = false;557 }558 559 if (cparams.auto_fgdn) {560 LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func);561 resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar);562 resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch);563 cparams.auto_fgdn = false;564 }565 566 if (cparams.auto_flid) {567 LLAMA_LOG_INFO("%s: resolving fused Lightning Indexer support:\n", func);568 resolve(llm_fused_op_lid_probe, cparams.fused_lid);569 cparams.auto_flid = false;570 }571 572 if (cparams.auto_fhc) {573 LLAMA_LOG_INFO("%s: resolving fused DeepSeek V4 HC support:\n", func);574 resolve(llm_fused_op_dsv4_hc_pre_probe, cparams.fused_dsv4_hc_pre);575 resolve(llm_fused_op_dsv4_hc_comb_probe, cparams.fused_dsv4_hc_comb);576 resolve(llm_fused_op_dsv4_hc_post_probe, cparams.fused_dsv4_hc_post);577 cparams.auto_fhc = false;578 }579}580 581void llama_context::sched_reserve() {582 if (!sched_need_reserve) {583 return;584 }585 586 sched_need_reserve = false;587 588 LLAMA_LOG_INFO("%s: reserving ...\n", __func__);589 590 synchronize();591 592 const int64_t t_start_us = ggml_time_us();593 594 const uint32_t n_seqs = cparams.n_seq_max;595 const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);596 597 const size_t max_nodes = this->graph_max_nodes(n_tokens);598 599 LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes);600 601 gf_res_prev.reset(new llm_graph_result(max_nodes));602 gf_res_reserve.reset(new llm_graph_result(max_nodes));603 604 sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));605 606 llama_memory_context_ptr mctx;607 if (memory) {608 LLAMA_LOG_DEBUG("%s: reserving full memory module\n", __func__);609 mctx = memory->init_full();610 if (!mctx) {611 throw std::runtime_error("failed to initialize memory module");612 }613 }614 615 // avoid reserving graphs with zero outputs - assume one output per sequence616 const int n_outputs = n_seqs;617 618 LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);619 620 resolve_fused_ops(mctx.get(), n_seqs);621 622 // reserve worst-case graph623 int n_splits_pp = -1;624 int n_nodes_pp = -1;625 626 int n_splits_tg = -1;627 int n_nodes_tg = -1;628 629 const uint32_t n_outputs_pp = std::min(n_tokens, cparams.n_outputs_max);630 631 // reserve pp (prompt processing) graph first so that buffers are only allocated once632 {633 auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),634 model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);635 if (!gf) {636 if (cparams.pipeline_parallel) {637 LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);638 cparams.pipeline_parallel = false;639 sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));640 gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());641 }642 if (!gf) {643 throw std::runtime_error("failed to allocate compute pp buffers");644 }645 }646 647 n_splits_pp = ggml_backend_sched_get_n_splits(sched.get());648 n_nodes_pp = ggml_graph_n_nodes(gf);649 }650 651 // reserve with tg (token generation) graph to get the number of splits and nodes652 {653 auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);654 if (!gf) {655 throw std::runtime_error("failed to allocate compute tg buffers");656 }657 658 n_splits_tg = ggml_backend_sched_get_n_splits(sched.get());659 n_nodes_tg = ggml_graph_n_nodes(gf);660 }661 662 // reserve again with pp graph to avoid ggml-alloc reallocations during inference663 {664 // TODO: not sure if the following graph would be worst case for multi-stream KV caches:665 //666 // auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());667 //668 auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);669 if (!gf) {670 throw std::runtime_error("failed to allocate compute pp buffers");671 }672 }673 674 for (size_t i = 0; i < backend_ptrs.size(); ++i) {675 ggml_backend_t backend = backend_ptrs[i];676 ggml_backend_buffer_type_t buft = backend_buft[i];677 if (!model.hparams.no_alloc) {678 backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);679 }680 if (backend_buf_exp_size[i] > 1) {681 LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,682 ggml_backend_buft_name(buft),683 backend_buf_exp_size[i] / 1024.0 / 1024.0);684 }685 }686 687 if (n_nodes_pp == n_nodes_tg) {688 LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp);689 } else {690 LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg);691 }692 693 if (n_splits_pp == n_splits_tg) {694 LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp);695 } else {696 LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg);697 }698 699 const int64_t t_end_us = ggml_time_us();700 701 LLAMA_LOG_INFO("%s: reserve took %.2f ms, sched copies = %d\n",702 __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get()));703}704 705void llama_context::synchronize() {706 if (!sched) {707 return;708 }709 710 ggml_backend_sched_synchronize(sched.get());711 712 // FIXME: if multiple single tokens are evaluated without a synchronization,713 // the stats will be added to the prompt evaluation stats714 // this should only happen when using batch size 1 to evaluate a batch715 716 // add the evaluation to the stats717 if (n_queued_tokens == 1) {718 if (!cparams.no_perf) {719 t_eval_us += ggml_time_us() - t_compute_start_us;720 }721 n_eval++;722 } else if (n_queued_tokens > 1) {723 if (!cparams.no_perf) {724 t_p_eval_us += ggml_time_us() - t_compute_start_us;725 }726 n_p_eval += n_queued_tokens;727 }728 729 // get a more accurate load time, upon first eval730 if (n_queued_tokens > 0 && !has_evaluated_once) {731 t_load_us = ggml_time_us() - t_start_us;732 has_evaluated_once = true;733 }734 735 n_queued_tokens = 0;736 t_compute_start_us = 0;737}738 739const llama_model & llama_context::get_model() const {740 return model;741}742 743const llama_cparams & llama_context::get_cparams() const {744 return cparams;745}746 747ggml_backend_sched_t llama_context::get_sched() const {748 return sched.get();749}750 751uint32_t llama_context::n_ctx() const {752 return cparams.n_ctx;753}754 755uint32_t llama_context::n_ctx_seq() const {756 return cparams.n_ctx_seq;757}758 759uint32_t llama_context::n_batch() const {760 return cparams.n_batch;761}762 763uint32_t llama_context::n_ubatch() const {764 return cparams.n_ubatch;765}766 767uint32_t llama_context::n_seq_max() const {768 return cparams.n_seq_max;769}770 771uint32_t llama_context::n_threads() const {772 return cparams.n_threads;773}774 775uint32_t llama_context::n_threads_batch() const {776 return cparams.n_threads_batch;777}778 779llama_memory_t llama_context::get_memory() const {780 return memory.get();781}782 783bool llama_context::memory_update(bool optimize) {784 if (!memory) {785 return false;786 }787 788 {789 const auto mctx = memory->init_update(this, optimize);790 switch (mctx->get_status()) {791 case LLAMA_MEMORY_STATUS_SUCCESS:792 {793 // noop794 } break;795 case LLAMA_MEMORY_STATUS_NO_UPDATE:796 {797 // no updates need to be performed798 return false;799 }800 case LLAMA_MEMORY_STATUS_FAILED_PREPARE:801 case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:802 {803 LLAMA_LOG_ERROR("%s: failed to prepare memory update\n", __func__);804 return false;805 }806 }807 808 // reset the previous graph result to make sure that it won't be reused809 // TODO: change the mctx->apply() to return information if a graph reserve is needed810 // reset the graph result only if the memory module did reset the scheduler811 gf_res_prev->reset();812 813 if (!mctx->apply()) {814 LLAMA_LOG_ERROR("%s: failed to apply memory update\n", __func__);815 }816 }817 818 // if the memory module did any computation, we have to reserve a new worst-case graph819 {820 const auto mctx = memory->init_full();821 if (!mctx) {822 throw std::runtime_error("failed to initialize memory context");823 }824 825 const uint32_t n_seqs = cparams.n_seq_max;826 const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);827 828 const uint32_t n_outputs_max = std::min(n_tokens, cparams.n_outputs_max);829 830 auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_max, mctx.get());831 if (!gf) {832 LLAMA_LOG_ERROR("%s: failed to reserve graph after the memory update\n", __func__);833 }834 }835 836 return true;837}838 839enum llama_pooling_type llama_context::pooling_type() const {840 return cparams.pooling_type;841}842 843float * llama_context::get_logits() {844 output_reorder();845 846 return logits.data;847}848 849int64_t llama_context::output_resolve_row(int32_t i) const {850 int64_t j = -1;851 852 // support negative indices (last output row)853 if (i < 0) {854 j = n_outputs + i;855 if (j < 0) {856 throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs));857 }858 } else if ((size_t) i >= output_ids.size()) {859 throw std::runtime_error(format("out of range [0, %zu)", output_ids.size()));860 } else {861 // use output_ids to translate the batch token index into a row number862 // that holds this token's data.863 j = output_ids[i];864 }865 866 if (j < 0) {867 // the batch token was not configured to output anything868 throw std::runtime_error(format("batch.logits[%d] != true", i));869 }870 871 if (j >= n_outputs) {872 throw std::runtime_error(format("corrupt output buffer (j=%" PRId64 ", n_outputs=%d)", j, n_outputs));873 }874 875 return j;876}877 878float * llama_context::get_logits_ith(int32_t i) {879 output_reorder();880 881 try {882 if (logits.data == nullptr) {883 throw std::runtime_error("no logits");884 }885 886 const int64_t j = output_resolve_row(i);887 return logits.data + j*model.vocab.n_tokens();888 } catch (const std::exception & err) {889 LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what());890#ifndef NDEBUG891 GGML_ABORT("fatal error");892#else893 return nullptr;894#endif895 }896}897 898float * llama_context::get_embeddings() {899 output_reorder();900 901 return embd.data;902}903 904llama_token * llama_context::get_sampled_tokens() const{905 return sampling.sampled.data;906}907 908float * llama_context::get_embeddings_ith(int32_t i) {909 output_reorder();910 911 try {912 if (embd.data == nullptr) {913 throw std::runtime_error("no embeddings");914 }915 916 const int64_t j = output_resolve_row(i);917 const uint32_t n_embd_out = model.hparams.n_embd_out();918 return embd.data + j*n_embd_out;919 } catch (const std::exception & err) {920 LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what());921#ifndef NDEBUG922 GGML_ABORT("fatal error");923#else924 return nullptr;925#endif926 }927}928 929float * llama_context::get_embeddings_seq(llama_seq_id seq_id) {930 auto it = embd_seq.find(seq_id);931 if (it == embd_seq.end()) {932 return nullptr;933 }934 935 return it->second.data();936}937 938float * llama_context::get_embeddings_nextn() {939 output_reorder();940 941 return embd_nextn.data;942}943 944float * llama_context::get_embeddings_nextn_ith(int32_t i) {945 output_reorder();946 947 try {948 if (embd_nextn.data == nullptr) {949 throw std::runtime_error("no nextn embeddings");950 }951 952 const uint32_t n_embd = model.hparams.n_embd_out();953 954 if (!cparams.embeddings_nextn_masked) {955 // unmasked: nextn rows are stored densely, indexed by raw token position.956 if (i < 0 || (size_t)(i + 1) * n_embd > embd_nextn.size) {957 throw std::runtime_error(format("out of range [0, %zu)", embd_nextn.size / n_embd));958 }959 return embd_nextn.data + (size_t) i * n_embd;960 }961 962 const int64_t j = output_resolve_row(i);963 return embd_nextn.data + j*n_embd;964 } catch (const std::exception & err) {965 LLAMA_LOG_ERROR("%s: invalid nextn embeddings id %d, reason: %s\n", __func__, i, err.what());966#ifndef NDEBUG967 GGML_ABORT("fatal error");968#else969 return nullptr;970#endif971 }972}973 974float * llama_context::get_embeddings_layer_inp(uint32_t lid) {975 output_reorder();976 977 GGML_ASSERT(lid < embd_layer_inp.size() && embd_layer_inp[lid].has_data());978 979 return embd_layer_inp[lid].data;980}981 982llama_token llama_context::get_sampled_token_ith(int32_t idx) {983 output_reorder();984 985 if (!sampling.sampled.has_data()) {986 return LLAMA_TOKEN_NULL;987 }988 989 try {990 const int64_t row = output_resolve_row(idx);991 GGML_ASSERT(row < (int64_t) sampling.sampled.size);992 return sampling.sampled.data[row];993 } catch (const std::exception & err) {994 LLAMA_LOG_ERROR("%s: invalid backend sampled token id %d, reason: %s\n", __func__, idx, err.what());995 return LLAMA_TOKEN_NULL;996 }997}998 999float * llama_context::get_sampled_probs_ith(int32_t idx) {1000 output_reorder();1001 1002 if (!sampling.probs.has_data()) {1003 return nullptr;1004 }1005 1006 try {1007 const int64_t row = output_resolve_row(idx);1008 if ((size_t) row >= sampling.probs_count.size() || sampling.probs_count[row] == 0) {1009 return nullptr;1010 }1011 return sampling.probs.data + row*model.vocab.n_tokens();1012 } catch (const std::exception & err) {1013 LLAMA_LOG_ERROR("%s: invalid backend sampled probs id %d, reason: %s\n", __func__, idx, err.what());1014 return nullptr;1015 }1016}1017 1018float * llama_context::get_sampled_logits_ith(int32_t idx) {1019 output_reorder();1020 1021 if (!sampling.logits.has_data()) {1022 return nullptr;1023 }1024 1025 try {1026 const int64_t row = output_resolve_row(idx);1027 if ((size_t) row >= sampling.logits_count.size() || sampling.logits_count[row] == 0) {1028 return nullptr;1029 }1030 return sampling.logits.data + row*model.vocab.n_tokens();1031 } catch (const std::exception & err) {1032 LLAMA_LOG_ERROR("%s: invalid backend sampled logits id %d, reason: %s\n", __func__, idx, err.what());1033 return nullptr;1034 }1035}1036 1037const llama_token * llama_context::get_sampled_candidates_ith(int32_t idx) {1038 output_reorder();1039 1040 try {1041 const int64_t row = output_resolve_row(idx);1042 if (sampling.candidates.has_data() &&1043 (size_t) row < sampling.candidates_count.size() &&1044 sampling.candidates_count[row] > 0) {1045 return sampling.candidates.data + row*model.vocab.n_tokens();1046 }1047 } catch (const std::exception & err) {1048 // fallback to full vocab list1049 GGML_UNUSED(err);1050 }1051 1052 return sampling.token_ids_full_vocab.data();1053}1054 1055size_t llama_context::get_sampled_candidates_count(int32_t idx) {1056 output_reorder();1057 1058 if (!sampling.candidates.has_data()) {1059 return 0;1060 }1061 1062 try {1063 const int64_t row = output_resolve_row(idx);1064 if ((size_t) row >= sampling.candidates_count.size()) {1065 return 0;1066 }1067 return sampling.candidates_count[row];1068 } catch (const std::exception & err) {1069 LLAMA_LOG_ERROR("%s: invalid backend sampled candidates count id %d, reason: %s\n", __func__, idx, err.what());1070 return 0;1071 }1072}1073 1074size_t llama_context::get_sampled_logits_count(int32_t idx) {1075 output_reorder();1076 1077 if (!sampling.logits.has_data()) {1078 return model.vocab.n_tokens();1079 }1080 1081 try {1082 const int64_t row = output_resolve_row(idx);1083 if ((size_t) row >= sampling.logits_count.size()) {1084 return 0;1085 }1086 return sampling.logits_count[row];1087 } catch (const std::exception & err) {1088 LLAMA_LOG_ERROR("%s: invalid backend sampled logits count id %d, reason: %s\n", __func__, idx, err.what());1089 return 0;1090 }1091}1092 1093size_t llama_context::get_sampled_probs_count(int32_t idx) {1094 output_reorder();1095 1096 if (!sampling.probs.has_data()) {1097 return 0;1098 }1099 1100 try {1101 const int64_t row = output_resolve_row(idx);1102 if ((size_t) row >= sampling.probs_count.size()) {1103 return 0;1104 }1105 return sampling.probs_count[row];1106 } catch (const std::exception & err) {1107 LLAMA_LOG_ERROR("%s: invalid backend sampled probs count id %d, reason: %s\n", __func__, idx, err.what());1108 return 0;1109 }1110}1111 1112 1113void llama_context::attach_threadpool(1114 ggml_threadpool_t threadpool,1115 ggml_threadpool_t threadpool_batch) {1116 LLAMA_LOG_DEBUG("%s: call\n", __func__);1117 1118 this->threadpool = threadpool;1119 this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;1120}1121 1122void llama_context::detach_threadpool() {1123 LLAMA_LOG_DEBUG("%s: call\n", __func__);1124 1125 this->threadpool = nullptr;1126 this->threadpool_batch = nullptr;1127}1128 1129void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) {1130 LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch);1131 1132 cparams.n_threads = n_threads;1133 cparams.n_threads_batch = n_threads_batch;1134}1135 1136void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) {1137 LLAMA_LOG_DEBUG("%s: call\n", __func__);1138 1139 this->abort_callback = abort_callback;1140 this->abort_callback_data = abort_callback_data;1141 1142 for (auto & backend : backends) {1143 auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));1144 if (reg) {1145 auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");1146 if (set_abort_callback_fn) {1147 set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);1148 }1149 }1150 }1151}1152 1153void llama_context::set_embeddings(bool value) {1154 LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);1155 1156 cparams.embeddings = value;1157 1158 // TODO: not sure yet if we want to reserve here1159 //sched_need_reserve = true;1160}1161 1162void llama_context::set_embeddings_nextn(bool value, bool masked) {1163 LLAMA_LOG_DEBUG("%s: value = %d, masked = %d\n", __func__, value, masked);1164 1165 cparams.embeddings_nextn = value;1166 cparams.embeddings_nextn_masked = masked;1167}1168 1169void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) {1170 LLAMA_LOG_DEBUG("%s: lid = %d, enable = %d\n", __func__, lid, enable);1171 1172 GGML_ASSERT(lid <= model.hparams.n_layer());1173 1174 cparams.embeddings_layer_inp[lid] = enable;1175 1176 // note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected1177 sched_need_reserve = true;1178}1179 1180void llama_context::set_nextn_layer_offset(int32_t offset) {1181 cparams.nextn_layer_offset = offset;1182}1183 1184void llama_context::set_causal_attn(bool value) {1185 LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);1186 1187 if (cparams.causal_attn == value) {1188 return;1189 }1190 1191 cparams.causal_attn = value;1192 1193 sched_need_reserve = true;1194}1195 1196void llama_context::set_warmup(bool value) {1197 LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);1198 1199 if (cparams.warmup == value) {1200 return;