KBaba7/llama.cpp

1#include "arg.h"2#include "common.h"3#include "log.h"4#include "llama.h"5 6#include <algorithm>7#include <cstdio>8#include <string>9#include <vector>10 11static void print_usage(int, char ** argv) {12    LOG("\nexample usage:\n");13    LOG("\n    %s -m model.gguf -c 2048 -b 2048 -ub 512 -npp 128,256,512 -ntg 128,256 -npl 1,2,4,8,16,32 [-pps]\n", argv[0]);14    LOG("\n");15}16 17int main(int argc, char ** argv) {18    common_params params;19 20    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_BENCH, print_usage)) {21        return 1;22    }23 24    common_init();25 26    int is_pp_shared = params.is_pp_shared;27 28    std::vector<int> n_pp = params.n_pp;29    std::vector<int> n_tg = params.n_tg;30    std::vector<int> n_pl = params.n_pl;31 32    // init LLM33 34    llama_backend_init();35    llama_numa_init(params.numa);36 37    // initialize the model38 39    llama_model_params model_params = common_model_params_to_llama(params);40 41    llama_model * model = llama_model_load_from_file(params.model.c_str(), model_params);42 43    if (model == NULL) {44        fprintf(stderr , "%s: error: unable to load model\n" , __func__);45        return 1;46    }47 48    llama_context_params ctx_params = common_context_params_to_llama(params);49 50    // ensure enough sequences are available51    ctx_params.n_seq_max = n_pl.empty() ? 1 : *std::max_element(n_pl.begin(), n_pl.end());52 53    llama_context * ctx = llama_init_from_model(model, ctx_params);54 55    if (ctx == NULL) {56        fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);57        return 1;58    }59 60    const int32_t n_kv_max = llama_n_ctx(ctx);61 62    llama_batch batch = llama_batch_init(n_kv_max, 0, 1);63 64    // decode in batches of ctx_params.n_batch tokens65    auto decode_helper = [](llama_context * ctx, llama_batch & batch, int32_t n_batch) {66        for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch) {67            const int32_t n_tokens = std::min(n_batch, (int32_t) (batch.n_tokens - i));68 69            llama_batch batch_view = {70                n_tokens,71                batch.token    + i,72                nullptr,73                batch.pos      + i,74                batch.n_seq_id + i,75                batch.seq_id   + i,76                batch.logits   + i,77            };78 79            const int ret = llama_decode(ctx, batch_view);80            if (ret != 0) {81                LOG_ERR("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret);82                return false;83            }84 85            llama_synchronize(ctx);86        }87 88        return true;89    };90 91    // warm up92    {93        for (int i = 0; i < 16; ++i) {94            common_batch_add(batch, 0, i, { 0 }, false);95        }96 97        if (!decode_helper(ctx, batch, ctx_params.n_batch)) {98            LOG_ERR("%s: llama_decode() failed\n", __func__);99            return 1;100        }101    }102 103    if (!params.batched_bench_output_jsonl) {104        LOG("\n");105        LOG("%s: n_kv_max = %d, n_batch = %d, n_ubatch = %d, flash_attn = %d, is_pp_shared = %d, n_gpu_layers = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.is_pp_shared, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch);106        LOG("\n");107        LOG("|%6s | %6s | %4s | %6s | %8s | %8s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "B", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s", "T s", "S t/s");108        LOG("|%6s-|-%6s-|-%4s-|-%6s-|-%8s-|-%8s-|-%8s-|-%8s-|-%8s-|-%8s-|\n", "------", "------", "----", "------", "--------", "--------", "--------", "--------", "--------", "--------");109    }110 111    for (        int i_pp = 0; i_pp < (int) n_pp.size(); ++i_pp) {112        for (    int i_tg = 0; i_tg < (int) n_tg.size(); ++i_tg) {113            for (int i_pl = 0; i_pl < (int) n_pl.size(); ++i_pl) {114                const int pp = n_pp[i_pp];115                const int tg = n_tg[i_tg];116                const int pl = n_pl[i_pl];117 118                const int n_ctx_req = is_pp_shared ? pp + pl*tg : pl*(pp + tg);119 120                if (n_ctx_req > n_kv_max) {121                    continue;122                }123 124                common_batch_clear(batch);125 126                for (int i = 0; i < pp; ++i) {127                    for (int j = 0; j < (is_pp_shared ? 1 : pl); ++j) {128                        common_batch_add(batch, 0, i, { j }, false);129                    }130                }131                batch.logits[batch.n_tokens - 1] = true;132 133                const auto t_pp_start = ggml_time_us();134 135                llama_kv_cache_clear(ctx);136 137                if (!decode_helper(ctx, batch, ctx_params.n_batch)) {138                    LOG_ERR("%s: llama_decode() failed\n", __func__);139                    return 1;140                }141 142                if (is_pp_shared) {143                    for (int32_t i = 1; i < pl; ++i) {144                        llama_kv_cache_seq_cp(ctx, 0, i, -1, -1);145                    }146                }147 148                const auto t_pp_end = ggml_time_us();149 150                const auto t_tg_start = ggml_time_us();151 152                for (int i = 0; i < tg; ++i) {153                    common_batch_clear(batch);154 155                    for (int j = 0; j < pl; ++j) {156                        common_batch_add(batch, 0, pp + i, { j }, true);157                    }158 159                    if (!decode_helper(ctx, batch, ctx_params.n_batch)) {160                        LOG_ERR("%s: llama_decode() failed\n", __func__);161                        return 1;162                    }163                }164 165                const auto t_tg_end = ggml_time_us();166 167                const int32_t n_kv = n_ctx_req;168 169                const float t_pp = (t_pp_end - t_pp_start) / 1000000.0f;170                const float t_tg = (t_tg_end - t_tg_start) / 1000000.0f;171                const float t    = t_pp + t_tg;172 173                const float speed_pp = is_pp_shared ? pp / t_pp : pl*pp / t_pp;174                const float speed_tg = pl*tg / t_tg;175                const float speed    = n_kv / t;176 177                if(params.batched_bench_output_jsonl) {178                    LOG(179                        "{\"n_kv_max\": %d, \"n_batch\": %d, \"n_ubatch\": %d, \"flash_attn\": %d, \"is_pp_shared\": %d, \"n_gpu_layers\": %d, \"n_threads\": %u, \"n_threads_batch\": %u, "180                        "\"pp\": %d, \"tg\": %d, \"pl\": %d, \"n_kv\": %d, \"t_pp\": %f, \"speed_pp\": %f, \"t_tg\": %f, \"speed_tg\": %f, \"t\": %f, \"speed\": %f}\n",181                        n_kv_max, params.n_batch, params.n_ubatch, params.flash_attn, params.is_pp_shared, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch,182                        pp, tg, pl, n_kv, t_pp, speed_pp, t_tg, speed_tg, t, speed183                    );184                } else {185                    LOG("|%6d | %6d | %4d | %6d | %8.3f | %8.2f | %8.3f | %8.2f | %8.3f | %8.2f |\n", pp, tg, pl, n_kv, t_pp, speed_pp, t_tg, speed_tg, t, speed);186                }187            }188        }189    }190 191    LOG("\n");192    llama_perf_context_print(ctx);193 194    llama_batch_free(batch);195 196    llama_free(ctx);197    llama_model_free(model);198 199    llama_backend_free();200 201    LOG("\n\n");202 203    return 0;204}205