echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0604
1#include "llama-context.h"2 3#include "ggml.h"4#include "llama-arch.h"5#include "llama-impl.h"6#include "llama-batch.h"7#include "llama-io.h"8#include "llama-memory.h"9#include "llama-mmap.h"10#include "llama-model.h"11#include "llama-ext.h"12#include "llama.h"13 14#include <cinttypes>15#include <cmath>16#include <cstring>17#include <limits>18#include <stdexcept>19 20//21// llama_context22//23 24llama_context::llama_context(25 const llama_model & model,26 llama_context_params params) :27 model(model),28 cvec(std::make_unique<llama_adapter_cvec>()),29 loras(std::make_unique<llama_adapter_loras>()),30 balloc(std::make_unique<llama_batch_allocr>(model.hparams.n_pos_per_embd())) {31 // TODO warning when creating llama_context with awkward ctx size that is not a power of 2,32 // may need to be backend-dependent33 LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__);34 35 t_start_us = model.t_start_us;36 t_load_us = model.t_load_us;37 38 const auto & hparams = model.hparams;39 40 cparams.n_seq_max = std::max(1u, params.n_seq_max);41 if (cparams.n_seq_max > LLAMA_MAX_SEQ) {42 throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ));43 }44 45 cparams.n_threads = params.n_threads;46 cparams.n_threads_batch = params.n_threads_batch;47 cparams.yarn_ext_factor = params.yarn_ext_factor >= 0.0f ? params.yarn_ext_factor : hparams.yarn_ext_factor;48 cparams.yarn_attn_factor = params.yarn_attn_factor >= 0.0f ? params.yarn_attn_factor : hparams.yarn_attn_factor;49 cparams.yarn_beta_fast = params.yarn_beta_fast >= 0.0f ? params.yarn_beta_fast : hparams.yarn_beta_fast;50 cparams.yarn_beta_slow = params.yarn_beta_slow >= 0.0f ? params.yarn_beta_slow : hparams.yarn_beta_slow;51 cparams.embeddings = params.embeddings;52 cparams.offload_kqv = params.offload_kqv;53 cparams.no_perf = params.no_perf;54 cparams.pooling_type = params.pooling_type;55 cparams.warmup = false;56 57 cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;58 cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;59 cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;60 61 cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :62 hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :63 hparams.n_ctx_train;64 65 cparams.cb_eval = params.cb_eval;66 cparams.cb_eval_user_data = params.cb_eval_user_data;67 68 // Initialize backend samplers here so they are part of the sampling graph69 // before the reserve passes run later in this function. This avoids a later70 // re-reserve when graph nodes change.71 if (params.samplers != nullptr && params.n_samplers > 0) {72 for (size_t i = 0; i < params.n_samplers; ++i) {73 const auto & config = params.samplers[i];74 75 if (llama_sampler_chain_get(config.sampler, -1) == nullptr) {76 throw std::runtime_error("the backend samplers must be of type llama_sampler_chain");77 }78 79 if (set_sampler(config.seq_id, config.sampler)) {80 const int n_samplers = llama_sampler_chain_n(config.sampler);81 82 LLAMA_LOG_INFO("%s: setting backend sampler for seq_id %d (n = %d)\n", __func__, config.seq_id, n_samplers);83 }84 }85 }86 87 auto rope_scaling_type = params.rope_scaling_type;88 if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {89 rope_scaling_type = hparams.rope_scaling_type_train;90 }91 92 if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {93 cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none94 }95 96 if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'97 cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;98 }99 100 if (cparams.yarn_ext_factor != 0) {101 static auto get_mscale = [](float scale, float mscale) {102 return scale <= 1.0f ? 1.0f : (0.1f * mscale * logf(scale) + 1.0f);103 };104 105 const float factor = 1.0f / cparams.rope_freq_scale;106 107 // ref: https://github.com/huggingface/transformers/blob/6d00f6b0a5679c36510f203e4226e36f517c3032/src/transformers/modeling_rope_utils.py#L336-L348108 if (hparams.rope_yarn_log_mul != 0.0f) {109 // note: here we assume `mscale == 1.0f`110 // TODO: start reading the actual value of mscale and handle the case where it is not 1.0f111 float mscale = 1.0f;112 const float mscale_all_dims = hparams.rope_yarn_log_mul;113 114 // [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]115 // special-case DEEPSEEK v2:116 // https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/main/config.json#L42-L43117 if (model.arch == LLM_ARCH_DEEPSEEK2 && mscale_all_dims != 1.0f) {118 mscale = mscale_all_dims;119 }120 121 cparams.yarn_attn_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dims);122 123 LLAMA_LOG_WARN("%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n",124 __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims);125 } else {126 cparams.yarn_attn_factor = get_mscale(factor, 1.0f);127 }128 129 // when YARN is applied with yarn_ext_factor != 0.0f, we need to cancel this factor:130 // https://github.com/ggml-org/llama.cpp/blob/a81a569577cc38b32558958b048228150be63eae/ggml/src/ggml-cpu/ops.cpp#L5541-L5544131 //132 // ref: https://github.com/ggml-org/llama.cpp/discussions/7416133 // https://github.com/ggml-org/llama.cpp/pull/17945134 cparams.yarn_attn_factor *= 1.0f / (1.0f + 0.1f * logf(factor));135 }136 137 cparams.yarn_attn_factor *= hparams.rope_attn_factor;138 139 if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {140 if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {141 cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;142 } else {143 cparams.pooling_type = hparams.pooling_type;144 }145 }146 147 if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {148 cparams.causal_attn = hparams.causal_attn;149 } else {150 cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;151 }152 153 cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED;154 cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO;155 156 cparams.fused_gdn_ar = true;157 cparams.fused_gdn_ch = true;158 cparams.auto_fgdn = true;159 160 // with causal attention, the batch size is limited by the context size161 cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;162 163 cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);164 165 cparams.op_offload = params.op_offload;166 cparams.kv_unified = params.kv_unified;167 168 // initialized later169 cparams.pipeline_parallel = false;170 171 {172 const char * LLAMA_GRAPH_REUSE_DISABLE = getenv("LLAMA_GRAPH_REUSE_DISABLE");173 graph_reuse_disable = LLAMA_GRAPH_REUSE_DISABLE ? (atoi(LLAMA_GRAPH_REUSE_DISABLE) != 0) : graph_reuse_disable;174 175 if (graph_reuse_disable) {176 LLAMA_LOG_WARN("%s: graph reuse disabled\n", __func__);177 }178 }179 180 // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732181 cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);182 183 if (cparams.kv_unified) {184 cparams.n_ctx_seq = cparams.n_ctx;185 } else {186 cparams.n_ctx_seq = cparams.n_ctx / cparams.n_seq_max;187 cparams.n_ctx_seq = GGML_PAD(cparams.n_ctx_seq, 256);188 189 if (cparams.n_ctx_seq == 0) {190 throw std::runtime_error("n_ctx_seq == 0");191 }192 193 if (cparams.n_ctx != cparams.n_ctx_seq * cparams.n_seq_max) {194 cparams.n_ctx = cparams.n_ctx_seq * cparams.n_seq_max;195 LLAMA_LOG_WARN("%s: n_ctx is not divisible by n_seq_max - rounding down to %u\n", __func__, cparams.n_ctx);196 }197 }198 199 LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max);200 LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);201 LLAMA_LOG_INFO("%s: n_ctx_seq = %u\n", __func__, cparams.n_ctx_seq);202 LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);203 LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);204 LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn);205 LLAMA_LOG_INFO("%s: flash_attn = %s\n", __func__, llama_flash_attn_type_name(params.flash_attn_type));206 LLAMA_LOG_INFO("%s: kv_unified = %s\n", __func__, cparams.kv_unified ? "true" : "false");207 LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);208 LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);209 210 if (cparams.n_ctx_seq < hparams.n_ctx_train) {211 LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",212 __func__, cparams.n_ctx_seq, hparams.n_ctx_train);213 }214 215 if (cparams.n_ctx_seq > hparams.n_ctx_train) {216 LLAMA_LOG_WARN("%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n",217 __func__, cparams.n_ctx_seq, hparams.n_ctx_train);218 }219 220 if (!hparams.vocab_only) {221 // GPU backends222 for (const auto & dev : model.devices) {223 ggml_backend_t backend = ggml_backend_dev_init(dev.dev, nullptr);224 if (backend == nullptr) {225 throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev.dev)));226 }227 backends.emplace_back(backend);228 }229 230 // add ACCEL backends (such as BLAS)231 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {232 ggml_backend_dev_t dev = ggml_backend_dev_get(i);233 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {234 ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);235 if (backend == nullptr) {236 throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));237 }238 backends.emplace_back(backend);239 }240 }241 242 // add CPU backend243 backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);244 if (backend_cpu == nullptr) {245 throw std::runtime_error("failed to initialize CPU backend");246 }247 backends.emplace_back(backend_cpu);248 249 // create a list of the set_n_threads functions in the backends250 for (auto & backend : backends) {251 ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());252 ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;253 if (reg) {254 auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");255 if (ggml_backend_set_n_threads_fn) {256 set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn);257 }258 }259 }260 261 llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data);262 263 // graph outputs buffer264 {265 if (output_reserve(params.n_seq_max) < params.n_seq_max) {266 throw std::runtime_error("failed to reserve initial output buffer");267 }268 269 LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,270 ggml_backend_buffer_name (buf_output.get()),271 ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0);272 }273 }274 275 // init the memory module276 if (!hparams.vocab_only) {277 llama_memory_params params_mem = {278 /*.type_k =*/ params.type_k,279 /*.type_v =*/ params.type_v,280 /*.swa_full =*/ params.swa_full,281 };282 283 memory.reset(model.create_memory(params_mem, cparams));284 }285 286 // init backends287 if (!hparams.vocab_only) {288 LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__);289 290 backend_buft.clear();291 backend_ptrs.clear();292 backend_buf_exp_size.clear();293 294 for (auto & backend : backends) {295 auto * buft = ggml_backend_get_default_buffer_type(backend.get());296 auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));297 298 if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {299 // use the host buffer of the first device CPU for faster transfer of the intermediate state300 const auto & dev = model.devices[0];301 auto * host_buft = ggml_backend_dev_host_buffer_type(dev.dev);302 if (host_buft) {303 buft = host_buft;304 }305 }306 307 backend_buft.push_back(buft);308 backend_ptrs.push_back(backend.get());309 backend_buf_exp_size.push_back(0);310 }311 312 LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size());313 314 // TODO: move these checks to ggml_backend_sched315 // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary316 bool pipeline_parallel =317 model.n_devices() > 1 &&318 model.n_gpu_layers() > model.hparams.n_layer &&319 model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&320 cparams.offload_kqv &&321 !model.has_tensor_overrides();322 323 // pipeline parallelism requires support for async compute and events in all devices324 if (pipeline_parallel) {325 for (auto & backend : backends) {326 auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));327 if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) {328 // ignore CPU backend329 // TODO: should we ignore ACCEL types too?330 continue;331 }332 auto * dev = ggml_backend_get_device(backend.get());333 ggml_backend_dev_props props;334 ggml_backend_dev_get_props(dev, &props);335 if (!props.caps.async || !props.caps.events) {336 // device does not support async compute or events337 pipeline_parallel = false;338 break;339 }340 }341 }342 343 cparams.pipeline_parallel = pipeline_parallel;344 345 if (cparams.pipeline_parallel) {346 LLAMA_LOG_INFO("%s: pipeline parallelism enabled\n", __func__);347 }348 349 sched_reserve();350 351 if (!cparams.flash_attn) {352 if (ggml_is_quantized(params.type_v)) {353 throw std::runtime_error("quantized V cache was requested, but this requires Flash Attention");354 }355 }356 }357 358 // Initialize the full vocabulary token ids for backend samplers.359 {360 const int n_vocab = model.vocab.n_tokens();361 362 sampling.token_ids_full_vocab.resize(n_vocab);363 for (int i = 0; i < n_vocab; ++i) {364 sampling.token_ids_full_vocab[i] = i;365 }366 }367}368 369llama_context::~llama_context() {370 if (!model.hparams.no_alloc) {371 for (size_t i = 0; i < backend_ptrs.size(); ++i) {372 ggml_backend_t backend = backend_ptrs[i];373 ggml_backend_buffer_type_t buft = backend_buft[i];374 375 const size_t size_exp = backend_buf_exp_size[i];376 const size_t size_act = ggml_backend_sched_get_buffer_size(sched.get(), backend);377 if (size_exp == size_act) {378 LLAMA_LOG_DEBUG("%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n",379 __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0));380 } else {381 LLAMA_LOG_WARN("%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n",382 __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0));383 }384 }385 }386 ggml_opt_free(opt_ctx);387}388 389void llama_context::sched_reserve() {390 if (!sched_need_reserve) {391 return;392 }393 394 sched_need_reserve = false;395 396 LLAMA_LOG_INFO("%s: reserving ...\n", __func__);397 398 synchronize();399 400 const int64_t t_start_us = ggml_time_us();401 402 const uint32_t n_seqs = cparams.n_seq_max;403 const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);404 405 const size_t max_nodes = this->graph_max_nodes(n_tokens);406 407 LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes);408 409 gf_res_prev.reset(new llm_graph_result(max_nodes));410 gf_res_reserve.reset(new llm_graph_result(max_nodes));411 412 sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));413 414 llama_memory_context_ptr mctx;415 if (memory) {416 LLAMA_LOG_DEBUG("%s: reserving full memory module\n", __func__);417 mctx = memory->init_full();418 if (!mctx) {419 throw std::runtime_error("failed to initialize memory module");420 }421 }422 423 // avoid reserving graphs with zero outputs - assume one output per sequence424 const int n_outputs = n_seqs;425 426 LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);427 428 // resolve automatic Flash Attention use429 if (cparams.auto_fa) {430 auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);431 if (!gf) {432 throw std::runtime_error("failed to reserve graph for Flash Attention check");433 }434 435 const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN) + 1;436 bool fa_device_mismatch = false;437 for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {438 ggml_tensor * n = ggml_graph_node(gf, i);439 if (n->op != GGML_OP_FLASH_ATTN_EXT) {440 continue;441 }442 ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));443 444 // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer445 GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0);446 const int il = std::stoi(n->name + prefix_len);447 ggml_backend_dev_t device_kv = model.dev_layer(il);448 if (device_fa != device_kv) {449 LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor "450 "is assigned to device %s (usually due to missing support)\n",451 __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa));452 // FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways453 fa_device_mismatch = true;454 break;455 }456 }457 458 if (fa_device_mismatch) {459 cparams.flash_attn = false;460 LLAMA_LOG_WARN("%s: Flash Attention was auto, set to disabled\n", __func__);461 } else {462 cparams.flash_attn = true;463 LLAMA_LOG_INFO("%s: Flash Attention was auto, set to enabled\n", __func__);464 }465 466 cparams.auto_fa = false;467 }468 469 if (cparams.auto_fgdn) {470 LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", __func__);471 472 if (cparams.fused_gdn_ar) {473 auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);474 if (!gf) {475 throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (autoregressive)");476 }477 478 const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR) + 1;479 bool gdn_device_mismatch = false;480 for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {481 ggml_tensor * n = ggml_graph_node(gf, i);482 if (n->op != GGML_OP_GATED_DELTA_NET) {483 continue;484 }485 ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));486 487 GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0);488 const int il = std::stoi(n->name + prefix_len);489 ggml_backend_dev_t device_kv = model.dev_layer(il);490 if (device_gdn != device_kv) {491 LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "492 "is assigned to device %s (usually due to missing support)\n",493 __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));494 gdn_device_mismatch = true;495 break;496 }497 }498 499 if (gdn_device_mismatch) {500 cparams.fused_gdn_ar = false;501 LLAMA_LOG_WARN("%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n", __func__);502 } else {503 LLAMA_LOG_INFO("%s: fused Gated Delta Net (autoregressive) enabled\n", __func__);504 }505 }506 507 if (cparams.fused_gdn_ch) {508 // more than one token in the batch per sequence in order to take the chunked path509 // note: n_outputs must match n_tokens for embedding models with mean/rank pooling,510 // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies511 // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens,512 // the ggml_mul_mat assertion fails. this matches the pp reservation below (line ~553).513 const uint32_t n_tokens_ch = 16*n_seqs;514 auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true);515 if (!gf) {516 throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (chunked)");517 }518 519 const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH) + 1;520 bool gdn_device_mismatch = false;521 for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {522 ggml_tensor * n = ggml_graph_node(gf, i);523 if (n->op != GGML_OP_GATED_DELTA_NET) {524 continue;525 }526 ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));527 528 GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0);529 const int il = std::stoi(n->name + prefix_len);530 ggml_backend_dev_t device_kv = model.dev_layer(il);531 if (device_gdn != device_kv) {532 LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "533 "is assigned to device %s (usually due to missing support)\n",534 __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));535 gdn_device_mismatch = true;536 break;537 }538 }539 540 if (gdn_device_mismatch) {541 cparams.fused_gdn_ch = false;542 LLAMA_LOG_WARN("%s: fused Gated Delta Net (chunked) not supported, set to disabled\n", __func__);543 } else {544 LLAMA_LOG_INFO("%s: fused Gated Delta Net (chunked) enabled\n", __func__);545 }546 }547 548 cparams.auto_fgdn = false;549 }550 551 // reserve worst-case graph552 int n_splits_pp = -1;553 int n_nodes_pp = -1;554 555 int n_splits_tg = -1;556 int n_nodes_tg = -1;557 558 // reserve pp (prompt processing) graph first so that buffers are only allocated once559 {560 auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(),561 model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);562 if (!gf) {563 if (cparams.pipeline_parallel) {564 LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);565 cparams.pipeline_parallel = false;566 sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));567 gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get());568 }569 if (!gf) {570 throw std::runtime_error("failed to allocate compute pp buffers");571 }572 }573 574 n_splits_pp = ggml_backend_sched_get_n_splits(sched.get());575 n_nodes_pp = ggml_graph_n_nodes(gf);576 }577 578 // reserve with tg (token generation) graph to get the number of splits and nodes579 {580 auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);581 if (!gf) {582 throw std::runtime_error("failed to allocate compute tg buffers");583 }584 585 n_splits_tg = ggml_backend_sched_get_n_splits(sched.get());586 n_nodes_tg = ggml_graph_n_nodes(gf);587 }588 589 // reserve again with pp graph to avoid ggml-alloc reallocations during inference590 {591 // TODO: not sure if the following graph would be worst case for multi-stream KV caches:592 //593 // auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());594 //595 auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(), model.hparams.no_alloc);596 if (!gf) {597 throw std::runtime_error("failed to allocate compute pp buffers");598 }599 }600 601 for (size_t i = 0; i < backend_ptrs.size(); ++i) {602 ggml_backend_t backend = backend_ptrs[i];603 ggml_backend_buffer_type_t buft = backend_buft[i];604 if (!model.hparams.no_alloc) {605 backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);606 }607 if (backend_buf_exp_size[i] > 1) {608 LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,609 ggml_backend_buft_name(buft),610 backend_buf_exp_size[i] / 1024.0 / 1024.0);611 }612 }613 614 if (n_nodes_pp == n_nodes_tg) {615 LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp);616 } else {617 LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg);618 }619 620 if (n_splits_pp == n_splits_tg) {621 LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp);622 } else {623 LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg);624 }625 626 const int64_t t_end_us = ggml_time_us();627 628 LLAMA_LOG_INFO("%s: reserve took %.2f ms, sched copies = %d\n",629 __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get()));630}631 632void llama_context::synchronize() {633 if (!sched) {634 return;635 }636 637 ggml_backend_sched_synchronize(sched.get());638 639 // FIXME: if multiple single tokens are evaluated without a synchronization,640 // the stats will be added to the prompt evaluation stats641 // this should only happen when using batch size 1 to evaluate a batch642 643 // add the evaluation to the stats644 if (n_queued_tokens == 1) {645 if (!cparams.no_perf) {646 t_eval_us += ggml_time_us() - t_compute_start_us;647 }648 n_eval++;649 } else if (n_queued_tokens > 1) {650 if (!cparams.no_perf) {651 t_p_eval_us += ggml_time_us() - t_compute_start_us;652 }653 n_p_eval += n_queued_tokens;654 }655 656 // get a more accurate load time, upon first eval657 if (n_queued_tokens > 0 && !has_evaluated_once) {658 t_load_us = ggml_time_us() - t_start_us;659 has_evaluated_once = true;660 }661 662 n_queued_tokens = 0;663 t_compute_start_us = 0;664}665 666const llama_model & llama_context::get_model() const {667 return model;668}669 670const llama_cparams & llama_context::get_cparams() const {671 return cparams;672}673 674ggml_backend_sched_t llama_context::get_sched() const {675 return sched.get();676}677 678uint32_t llama_context::n_ctx() const {679 return cparams.n_ctx;680}681 682uint32_t llama_context::n_ctx_seq() const {683 return cparams.n_ctx_seq;684}685 686uint32_t llama_context::n_batch() const {687 return cparams.n_batch;688}689 690uint32_t llama_context::n_ubatch() const {691 return cparams.n_ubatch;692}693 694uint32_t llama_context::n_seq_max() const {695 return cparams.n_seq_max;696}697 698uint32_t llama_context::n_threads() const {699 return cparams.n_threads;700}701 702uint32_t llama_context::n_threads_batch() const {703 return cparams.n_threads_batch;704}705 706llama_memory_t llama_context::get_memory() const {707 return memory.get();708}709 710bool llama_context::memory_update(bool optimize) {711 if (!memory) {712 return false;713 }714 715 {716 const auto mctx = memory->init_update(this, optimize);717 switch (mctx->get_status()) {718 case LLAMA_MEMORY_STATUS_SUCCESS:719 {720 // noop721 } break;722 case LLAMA_MEMORY_STATUS_NO_UPDATE:723 {724 // no updates need to be performed725 return false;726 }727 case LLAMA_MEMORY_STATUS_FAILED_PREPARE:728 case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:729 {730 LLAMA_LOG_ERROR("%s: failed to prepare memory update\n", __func__);731 return false;732 }733 }734 735 // reset the previous graph result to make sure that it won't be reused736 // TODO: change the mctx->apply() to return information if a graph reserve is needed737 // reset the graph result only if the memory module did reset the scheduler738 gf_res_prev->reset();739 740 if (!mctx->apply()) {741 LLAMA_LOG_ERROR("%s: failed to apply memory update\n", __func__);742 }743 }744 745 // if the memory module did any computation, we have to reserve a new worst-case graph746 {747 const auto mctx = memory->init_full();748 if (!mctx) {749 throw std::runtime_error("failed to initialize memory context");750 }751 752 const uint32_t n_seqs = cparams.n_seq_max;753 const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);754 755 auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get());756 if (!gf) {757 LLAMA_LOG_ERROR("%s: failed to reserve graph after the memory update\n", __func__);758 }759 }760 761 return true;762}763 764enum llama_pooling_type llama_context::pooling_type() const {765 return cparams.pooling_type;766}767 768float * llama_context::get_logits() {769 output_reorder();770 771 return logits.data;772}773 774int64_t llama_context::output_resolve_row(int32_t i) const {775 int64_t j = -1;776 777 // support negative indices (last output row)778 if (i < 0) {779 j = n_outputs + i;780 if (j < 0) {781 throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs));782 }783 } else if ((size_t) i >= output_ids.size()) {784 throw std::runtime_error(format("out of range [0, %zu)", output_ids.size()));785 } else {786 // use output_ids to translate the batch token index into a row number787 // that holds this token's data.788 j = output_ids[i];789 }790 791 if (j < 0) {792 // the batch token was not configured to output anything793 throw std::runtime_error(format("batch.logits[%d] != true", i));794 }795 796 if (j >= n_outputs) {797 throw std::runtime_error(format("corrupt output buffer (j=%" PRId64 ", n_outputs=%d)", j, n_outputs));798 }799 800 return j;801}802 803float * llama_context::get_logits_ith(int32_t i) {804 output_reorder();805 806 try {807 if (logits.data == nullptr) {808 throw std::runtime_error("no logits");809 }810 811 const int64_t j = output_resolve_row(i);812 return logits.data + j*model.vocab.n_tokens();813 } catch (const std::exception & err) {814 LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what());815#ifndef NDEBUG816 GGML_ABORT("fatal error");817#else818 return nullptr;819#endif820 }821}822 823float * llama_context::get_embeddings() {824 output_reorder();825 826 return embd.data;827}828 829llama_token * llama_context::get_sampled_tokens() const{830 return sampling.sampled.data;831}832 833float * llama_context::get_embeddings_ith(int32_t i) {834 output_reorder();835 836 try {837 if (embd.data == nullptr) {838 throw std::runtime_error("no embeddings");839 }840 841 const int64_t j = output_resolve_row(i);842 const uint32_t n_embd_out = model.hparams.n_embd_out();843 return embd.data + j*n_embd_out;844 } catch (const std::exception & err) {845 LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what());846#ifndef NDEBUG847 GGML_ABORT("fatal error");848#else849 return nullptr;850#endif851 }852}853 854float * llama_context::get_embeddings_seq(llama_seq_id seq_id) {855 auto it = embd_seq.find(seq_id);856 if (it == embd_seq.end()) {857 return nullptr;858 }859 860 return it->second.data();861}862 863llama_token llama_context::get_sampled_token_ith(int32_t idx) {864 output_reorder();865 866 if (!sampling.sampled.has_data()) {867 return LLAMA_TOKEN_NULL;868 }869 870 try {871 const int64_t row = output_resolve_row(idx);872 GGML_ASSERT(row < (int64_t) sampling.sampled.size);873 return sampling.sampled.data[row];874 } catch (const std::exception & err) {875 LLAMA_LOG_ERROR("%s: invalid backend sampled token id %d, reason: %s\n", __func__, idx, err.what());876 return LLAMA_TOKEN_NULL;877 }878}879 880float * llama_context::get_sampled_probs_ith(int32_t idx) {881 output_reorder();882 883 if (!sampling.probs.has_data()) {884 return nullptr;885 }886 887 try {888 const int64_t row = output_resolve_row(idx);889 if ((size_t) row >= sampling.probs_count.size() || sampling.probs_count[row] == 0) {890 return nullptr;891 }892 return sampling.probs.data + row*model.vocab.n_tokens();893 } catch (const std::exception & err) {894 LLAMA_LOG_ERROR("%s: invalid backend sampled probs id %d, reason: %s\n", __func__, idx, err.what());895 return nullptr;896 }897}898 899float * llama_context::get_sampled_logits_ith(int32_t idx) {900 output_reorder();901 902 if (!sampling.logits.has_data()) {903 return nullptr;904 }905 906 try {907 const int64_t row = output_resolve_row(idx);908 if ((size_t) row >= sampling.logits_count.size() || sampling.logits_count[row] == 0) {909 return nullptr;910 }911 return sampling.logits.data + row*model.vocab.n_tokens();912 } catch (const std::exception & err) {913 LLAMA_LOG_ERROR("%s: invalid backend sampled logits id %d, reason: %s\n", __func__, idx, err.what());914 return nullptr;915 }916}917 918const llama_token * llama_context::get_sampled_candidates_ith(int32_t idx) {919 output_reorder();920 921 try {922 const int64_t row = output_resolve_row(idx);923 if (sampling.candidates.has_data() &&924 (size_t) row < sampling.candidates_count.size() &&925 sampling.candidates_count[row] > 0) {926 return sampling.candidates.data + row*model.vocab.n_tokens();927 }928 } catch (const std::exception & err) {929 // fallback to full vocab list930 GGML_UNUSED(err);931 }932 933 return sampling.token_ids_full_vocab.data();934}935 936size_t llama_context::get_sampled_candidates_count(int32_t idx) {937 output_reorder();938 939 if (!sampling.candidates.has_data()) {940 return 0;941 }942 943 try {944 const int64_t row = output_resolve_row(idx);945 if ((size_t) row >= sampling.candidates_count.size()) {946 return 0;947 }948 return sampling.candidates_count[row];949 } catch (const std::exception & err) {950 LLAMA_LOG_ERROR("%s: invalid backend sampled candidates count id %d, reason: %s\n", __func__, idx, err.what());951 return 0;952 }953}954 955size_t llama_context::get_sampled_logits_count(int32_t idx) {956 output_reorder();957 958 if (!sampling.logits.has_data()) {959 return model.vocab.n_tokens();960 }961 962 try {963 const int64_t row = output_resolve_row(idx);964 if ((size_t) row >= sampling.logits_count.size()) {965 return 0;966 }967 return sampling.logits_count[row];968 } catch (const std::exception & err) {969 LLAMA_LOG_ERROR("%s: invalid backend sampled logits count id %d, reason: %s\n", __func__, idx, err.what());970 return 0;971 }972}973 974size_t llama_context::get_sampled_probs_count(int32_t idx) {975 output_reorder();976 977 if (!sampling.probs.has_data()) {978 return 0;979 }980 981 try {982 const int64_t row = output_resolve_row(idx);983 if ((size_t) row >= sampling.probs_count.size()) {984 return 0;985 }986 return sampling.probs_count[row];987 } catch (const std::exception & err) {988 LLAMA_LOG_ERROR("%s: invalid backend sampled probs count id %d, reason: %s\n", __func__, idx, err.what());989 return 0;990 }991}992 993 994void llama_context::attach_threadpool(995 ggml_threadpool_t threadpool,996 ggml_threadpool_t threadpool_batch) {997 LLAMA_LOG_DEBUG("%s: call\n", __func__);998 999 this->threadpool = threadpool;1000 this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;1001}1002 1003void llama_context::detach_threadpool() {1004 LLAMA_LOG_DEBUG("%s: call\n", __func__);1005 1006 this->threadpool = nullptr;1007 this->threadpool_batch = nullptr;1008}1009 1010void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) {1011 LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch);1012 1013 cparams.n_threads = n_threads;1014 cparams.n_threads_batch = n_threads_batch;1015}1016 1017void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) {1018 LLAMA_LOG_DEBUG("%s: call\n", __func__);1019 1020 this->abort_callback = abort_callback;1021 this->abort_callback_data = abort_callback_data;1022 1023 for (auto & backend : backends) {1024 auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));1025 if (reg) {1026 auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");1027 if (set_abort_callback_fn) {1028 set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);1029 }1030 }1031 }1032}1033 1034void llama_context::set_embeddings(bool value) {1035 LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);1036 1037 cparams.embeddings = value;1038 1039 // TODO: not sure yet if we want to reserve here1040 //sched_need_reserve = true;1041}1042 1043void llama_context::set_causal_attn(bool value) {1044 LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);1045 1046 if (cparams.causal_attn == value) {1047 return;1048 }1049 1050 cparams.causal_attn = value;1051 1052 sched_need_reserve = true;1053}1054 1055void llama_context::set_warmup(bool value) {1056 LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);1057 1058 if (cparams.warmup == value) {1059 return;1060 }1061 1062 cparams.warmup = value;1063 1064 // warmups are usually with small batches, so no need to reserve1065 //sched_need_reserve = true;1066}1067 1068bool llama_context::set_sampler(llama_seq_id seq_id, llama_sampler * sampler) {1069 if (!sampler && sampling.samplers.count(seq_id) == 0) {1070 return true;1071 }1072 1073 LLAMA_LOG_DEBUG("%s: seq_id = %d, sampler = %p\n", __func__, (int) seq_id, (void *) sampler);1074 1075 const bool can_offload =1076 sampler &&1077 sampler->iface->backend_init &&1078 sampler->iface->backend_apply &&1079 llama_sampler_chain_n(sampler) > 0;1080 1081 if (sampler && can_offload) {1082 auto * buft = ggml_backend_dev_buffer_type(model.dev_output());1083 1084 sampler->iface->backend_init(sampler, buft);1085 1086 sampling.samplers[seq_id] = sampler;1087 1088 sched_need_reserve = true;1089 1090 return true;1091 }1092 1093 if (sampler && !can_offload) {1094 LLAMA_LOG_WARN("%s: sampler '%s' for seq_id = %d, cannot be offloaded to the backend\n", __func__, llama_sampler_name(sampler), seq_id);1095 1096 if (sampling.samplers.count(seq_id) > 0) {1097 sched_need_reserve = true;1098 }1099 1100 sampling.samplers.erase(seq_id);1101 1102 return false;1103 }1104 1105 sampling.samplers.erase(seq_id);1106 1107 sched_need_reserve = true;1108 1109 return true;1110}1111 1112void llama_context::set_adapters_lora(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) {1113 LLAMA_LOG_DEBUG("%s: adapters = %p\n", __func__, (void *) adapters);1114 1115 if (adapters_lora_are_same(adapters, n_adapters, scales)) {1116 return;1117 }1118 1119 loras.reset(new llama_adapter_loras());1120 1121 for (size_t i = 0; i < n_adapters; i ++) {1122 if (scales[i] != 0.0f) {1123 loras->insert({adapters[i], scales[i]});1124 }1125 }1126 1127 sched_need_reserve = true;1128}1129 1130bool llama_context::adapters_lora_are_same(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) {1131 LLAMA_LOG_DEBUG("%s: adapters = %p\n", __func__, (void *) adapters);1132 1133 // Adapters with a zero scale are never added to `loras`, so also ignore them for the comparison.1134 size_t n_non_zero = 0;1135 1136 for (size_t i = 0; i < n_adapters; i ++) {1137 if (scales[i] == 0.0f) {1138 continue;1139 }1140 n_non_zero++;1141 1142 auto it = loras->find(adapters[i]);1143 1144 if (it == loras->end() || it->second != scales[i]) {1145 return false;1146 }1147 }1148 1149 if (n_non_zero != loras->size()) {1150 return false;1151 }1152 1153 return true;1154}1155 1156bool llama_context::set_adapter_cvec(1157 const float * data,1158 size_t len,1159 int32_t n_embd,1160 int32_t il_start,1161 int32_t il_end) {1162 LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end);1163 1164 bool res = cvec->apply(model, data, len, n_embd, il_start, il_end);1165 1166 sched_need_reserve = true;1167 1168 return res;1169}1170 1171llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) {1172 if (mctx && !mctx->apply()) {1173 LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__);1174 ret = GGML_STATUS_FAILED;1175 return nullptr;1176 }1177 1178 auto * res = gf_res_prev.get();1179 auto * gf = res->get_gf();1180 1181 // the new graph parameters1182 // in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters1183 const auto gparams = graph_params(res, ubatch, mctx, gtype);1184 1185 if (!graph_reuse_disable && res->can_reuse(gparams)) {1186 //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__);1187 1188 // with pipeline parallelism, the previous graph_compute_async may still be running1189 // on the GPU. we must synchronize before set_inputs to avoid overwriting input tensors1190 // that the previous compute is still reading.1191 if (cparams.pipeline_parallel) {1192 ggml_backend_sched_synchronize(sched.get());1193 }1194 1195 n_reused++;1196 } else {1197 res->reset();1198 1199 ggml_backend_sched_reset(sched.get());1200 ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);