Felipe97/llama-cpp-compiled

1#include "arg.h"2#include "common.h"3#include "sampling.h"4#include "speculative.h"5#include "log.h"6#include "llama.h"7 8#include <algorithm>9#include <clocale>10#include <cstdio>11#include <cstring>12#include <random>13#include <set>14#include <string>15#include <vector>16 17#define SPEC_VOCAB_MAX_SIZE_DIFFERENCE  12818#define SPEC_VOCAB_CHECK_START_TOKEN_ID 519 20struct seq_draft {21    bool active   = false;22    bool drafting = false;23    bool skip     = false;24 25    int i_batch_dft = 0;26    std::vector<int> i_batch_tgt;27 28    std::vector<llama_token> tokens;29    std::vector<std::vector<llama_token_data>> dists;30 31    struct common_sampler * smpl = nullptr;32};33 34int main(int argc, char ** argv) {35    std::setlocale(LC_NUMERIC, "C");36 37    common_params params;38 39    // needed to get candidate probs even for temp <= 0.040    params.sampling.n_probs = 128;41 42    common_init();43 44    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SPECULATIVE)) {45        return 1;46    }47 48    if (params.n_predict < -1) {49        LOG_ERR("%s: --n-predict must be >= -1\n", __func__);50        return 1;51    }52 53    if (params.speculative.draft.mparams.path.empty()) {54        LOG_ERR("%s: --model-draft is required\n", __func__);55        return 1;56    }57 58    // max number of parallel drafting sequences (i.e. tree branches)59    const int n_seq_dft = params.n_parallel;60 61    const auto output_limits = common_speculative_get_output_limits(62            params.n_batch, params.n_parallel, params.speculative.draft.n_max);63    params.n_outputs_max = output_limits.total;64    params.n_outputs_max_per_seq = output_limits.per_seq;65 66    // probability threshold for splitting a draft branch (only for n_seq_dft > 1)67    const float p_draft_split = params.speculative.draft.p_split;68 69    std::default_random_engine rng(params.sampling.seed == LLAMA_DEFAULT_SEED ? std::random_device()() : params.sampling.seed);70    std::uniform_real_distribution<> u_dist;71 72    // init llama.cpp73    llama_backend_init();74    llama_numa_init(params.numa);75 76    llama_model * model_tgt = NULL;77    llama_model * model_dft = NULL;78 79    llama_context * ctx_tgt = NULL;80    llama_context * ctx_dft = NULL;81 82    // load the target model83    auto llama_init_tgt = common_init_from_params(params);84 85    model_tgt = llama_init_tgt->model();86    ctx_tgt   = llama_init_tgt->context();87 88    // load the draft model89    params.devices = params.speculative.draft.devices;90    params.model = params.speculative.draft.mparams;91    params.n_gpu_layers = params.speculative.draft.n_gpu_layers;92    params.n_outputs_max = params.n_parallel;93    params.n_outputs_max_per_seq = 1;94    if (params.speculative.draft.cpuparams.n_threads > 0) {95        params.cpuparams.n_threads = params.speculative.draft.cpuparams.n_threads;96    }97 98    params.cpuparams_batch.n_threads = params.speculative.draft.cpuparams_batch.n_threads;99    params.tensor_buft_overrides     = params.speculative.draft.tensor_buft_overrides;100 101    auto llama_init_dft = common_init_from_params(params);102 103    model_dft = llama_init_dft->model();104    ctx_dft   = llama_init_dft->context();105 106    const llama_vocab * vocab_tgt = llama_model_get_vocab(model_tgt);107    const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft);108 109    const bool vocab_type_tgt = llama_vocab_type(vocab_tgt);110    LOG_DBG("vocab_type tgt: %d\n", vocab_type_tgt);111 112    const bool vocab_type_dft = llama_vocab_type(vocab_dft);113    LOG_DBG("vocab_type dft: %d\n", vocab_type_dft);114 115    if (vocab_type_tgt != vocab_type_dft) {116        LOG_ERR("%s: draft model vocab type must match target model to use speculation but ", __func__);117        LOG_ERR("vocab_type_dft = %d while vocab_type_tgt = %d\n", vocab_type_dft, vocab_type_tgt);118        return 1;119    }120 121    if (llama_vocab_get_add_bos(vocab_tgt) != llama_vocab_get_add_bos(vocab_dft) ||122        (llama_vocab_get_add_bos(vocab_tgt) && llama_vocab_bos(vocab_tgt) != llama_vocab_bos(vocab_dft))) {123        LOG_ERR("%s: draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n",124                __func__,125                llama_vocab_get_add_bos(vocab_tgt), llama_vocab_get_add_bos(vocab_dft),126                llama_vocab_bos(vocab_tgt), llama_vocab_bos(vocab_dft));127        return 1;128    }129 130    if (llama_vocab_get_add_eos(vocab_tgt) != llama_vocab_get_add_eos(vocab_dft) ||131        (llama_vocab_get_add_eos(vocab_tgt) && llama_vocab_eos(vocab_tgt) != llama_vocab_eos(vocab_dft))) {132        LOG_ERR("%s: draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n",133                __func__,134                llama_vocab_get_add_eos(vocab_tgt), llama_vocab_get_add_eos(vocab_dft),135                llama_vocab_eos(vocab_tgt), llama_vocab_eos(vocab_dft));136        return 1;137    }138 139    {140        const int n_vocab_tgt = llama_vocab_n_tokens(vocab_tgt);141        const int n_vocab_dft = llama_vocab_n_tokens(vocab_dft);142        const int vocab_diff  = n_vocab_tgt > n_vocab_dft143            ? n_vocab_tgt - n_vocab_dft144            : n_vocab_dft - n_vocab_tgt;145 146        if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) {147            LOG_ERR("%s: draft model vocab must closely match target model to use speculation but ", __func__);148            LOG_ERR("target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n",149                    n_vocab_tgt, llama_vocab_n_tokens(vocab_dft), vocab_diff, SPEC_VOCAB_MAX_SIZE_DIFFERENCE);150            return 1;151        }152 153        for (int i = SPEC_VOCAB_CHECK_START_TOKEN_ID; i < std::min(n_vocab_tgt, n_vocab_dft); ++i) {154            const char * token_text_tgt = llama_vocab_get_text(vocab_tgt, i);155            const char * token_text_dft = llama_vocab_get_text(vocab_dft, i);156 157            if (std::strcmp(token_text_tgt, token_text_dft) != 0) {158                LOG_ERR("%s: draft model vocab must match target model to use speculation but ", __func__);159                LOG_ERR("token %d content differs - target '%s', draft '%s'\n", i,160                        common_token_to_piece(vocab_tgt, i).c_str(),161                        common_token_to_piece(vocab_dft, i).c_str());162                return 1;163            }164        }165    }166 167    auto * mem_tgt = llama_get_memory(ctx_tgt);168    auto * mem_dft = llama_get_memory(ctx_dft);169 170    // Tokenize the prompt171    std::vector<llama_token> inp;172    inp = common_tokenize(ctx_tgt, params.prompt, true, true);173 174    const int max_context_size     = llama_n_ctx(ctx_tgt);175    const int max_tokens_list_size = max_context_size - 4;176 177    if ((int) inp.size() > max_tokens_list_size) {178        LOG_ERR("%s: prompt too long (%d tokens, max %d)\n", __func__, (int) inp.size(), max_tokens_list_size);179        return 1;180    }181 182    LOG("\n\n");183 184    for (auto id : inp) {185        LOG("%s", common_token_to_piece(ctx_tgt, id).c_str());186    }187 188    const int n_input = inp.size();189 190    const auto t_enc_start = ggml_time_us();191 192    // eval the prompt with both models193    llama_decode(ctx_tgt, llama_batch_get_one( inp.data(), n_input - 1));194    llama_decode(ctx_tgt, llama_batch_get_one(&inp.back(),           1));195    llama_decode(ctx_dft, llama_batch_get_one( inp.data(), n_input));196 197    const auto t_enc_end = ggml_time_us();198 199    // the 2 models should have the same vocab200    //GGML_ASSERT(n_vocab == llama_vocab_n_tokens(model_dft));201 202    // how many tokens to draft each time203    int n_draft = params.speculative.draft.n_max;204 205    int n_predict = 0;206    int n_drafted = 0;207    int n_accept  = 0;208 209    int n_past_tgt = inp.size();210    int n_past_dft = inp.size();211 212    // used to determine end of generation213    bool has_eos = false;214 215    // target model sampling context (reuse the llama_context's sampling instance)216    struct common_sampler * smpl = common_sampler_init(model_tgt, params.sampling);217 218    // draft sequence data219    std::vector<seq_draft> drafts(n_seq_dft);220 221    for (int s = 0; s < n_seq_dft; ++s) {222        // allocate llama_sampler for each draft sequence223        drafts[s].smpl = common_sampler_init(model_dft, params.sampling);224    }225 226    llama_batch batch_dft = llama_batch_init(llama_n_batch(ctx_dft), 0, 1);227    llama_batch batch_tgt = llama_batch_init(llama_n_batch(ctx_tgt), 0, n_seq_dft);228 229    const auto t_dec_start = ggml_time_us();230 231    // sample from the last token of the prompt232    drafts[0].i_batch_tgt.resize(1);233    drafts[0].i_batch_tgt[0] = 0;234 235    while (true) {236        std::set<int> active_seqs = {};237 238        // print current draft sequences239        for (int s = 0; s < n_seq_dft; ++s) {240            if (!drafts[s].active) {241                continue;242            }243 244            active_seqs.insert(s);245            const auto & tokens = drafts[s].tokens;246 247            LOG_DBG("draft %d: %s\n", s, string_from(ctx_dft, tokens).c_str());248        }249 250        int i_dft  = 0;251        int s_keep = 0;252 253        llama_token token_id;254        std::string token_str;255 256        // loop until we fail to accept a drafted token or we run out of drafted tokens257        while (true) {258 259            // check if the target token matches any of the drafts260            // for stochastic sampling, attempt to match the token with the drafted tokens261            {262                bool accept = false;263                if (params.sampling.temp > 0) {264                    // stochastic verification265                    common_sampler_sample(smpl, ctx_tgt, drafts[s_keep].i_batch_tgt[i_dft], true);266 267                    auto & dist_tgt = *common_sampler_get_candidates(smpl, true);268 269                    float p_tgt = 0.0f;270                    float p_dft = 0.0f;271 272                    while (active_seqs.size() > 0) {273                        // randomly select a sequence to verify from active sequences274                        std::uniform_int_distribution<unsigned int> u_int_dist(0, active_seqs.size() - 1);275                        int s = *std::next(active_seqs.begin(), u_int_dist(rng));276                        if (i_dft >= (int) drafts[s].tokens.size()) {277                            drafts[s].active = false;278                            active_seqs.erase(s);279                            continue;280                        }281                        if (accept) {282                            // if we already accepted a token, we can skip the rest283                            if (drafts[s].tokens[i_dft] != drafts[s_keep].tokens[i_dft]) {284                                drafts[s].active = false;285                                active_seqs.erase(s);286                            }287                            continue;288                        }289 290                        LOG_DBG("verifying sequence #%d at pos #%d from %d active sequence(s)\n", s, i_dft, (int) active_seqs.size());291                        float r = u_dist(rng);292                        llama_token_data_array dist_dft = { drafts[s].dists[i_dft].data() , drafts[s].dists[i_dft].size(), LLAMA_TOKEN_NULL, true };293 294                        //GGML_ASSERT(dist_tgt.size <= dist_dft.size);295 296                        // acquire the token probabilities assigned by the draft and target models297                        for (size_t i = 0; i < dist_tgt.size; i++) {298                            if (dist_tgt.data[i].id == drafts[s].tokens[i_dft]) {299                                p_tgt = dist_tgt.data[i].p;300                                break;301                            }302                        }303                        for (size_t i = 0; i < dist_dft.size; i++) {304                            if (dist_dft.data[i].id == drafts[s].tokens[i_dft]) {305                                p_dft = dist_dft.data[i].p;306                                break;307                            }308                        }309                        LOG_DBG("r = %f, p_dft = %f, p_tgt = %f\n", r, p_dft, p_tgt);310                        if (r <= p_tgt / p_dft) {311                            s_keep = s;312                            accept = true;313                            token_id = drafts[s].tokens[i_dft];314                            token_str = common_token_to_piece(ctx_tgt, token_id);315                            common_sampler_accept(smpl, token_id, true);316 317                            LOG_DBG("draft token %d of sequence %d (%d, '%s') accepted\n", i_dft, s, token_id, token_str.c_str());318                            break;319                        } else {320                            LOG_DBG("draft token %d of sequence %d (%d, '%s') rejected\n", i_dft, s, drafts[s].tokens[i_dft], common_token_to_piece(ctx_tgt, drafts[s].tokens[i_dft]).c_str());321                            drafts[s].active = false;322 323                            // calculate residual probability324                            GGML_ASSERT(dist_tgt.sorted);325                            GGML_ASSERT(dist_dft.sorted);326 327                            // sort dist by id328                            std::sort(dist_tgt.data, dist_tgt.data + dist_tgt.size, [](const llama_token_data &a, const llama_token_data &b) {329                                return a.id < b.id;330                            });331                            std::sort(dist_dft.data, dist_dft.data + dist_dft.size, [](const llama_token_data &a, const llama_token_data &b) {332                                return a.id < b.id;333                            });334 335                            float sum_probs = 0.0f;336 337                            for (size_t i = 0; i < dist_tgt.size; i++) {338                                if (i < dist_dft.size) {339                                    dist_tgt.data[i].p = std::max(0.0f, dist_tgt.data[i].p - dist_dft.data[i].p);340                                } else {341                                    dist_tgt.data[i].p = std::max(0.0f, dist_tgt.data[i].p);342                                }343 344                                sum_probs += dist_tgt.data[i].p;345                            }346 347                            for (size_t i = 0; i < dist_tgt.size; i++) {348                                dist_tgt.data[i].p /= sum_probs;349                            }350 351                            // sort dist_tgt by p desc352                            std::sort(dist_tgt.data, dist_tgt.data + dist_tgt.size, [](const llama_token_data &a, const llama_token_data &b) {353                                return a.p > b.p;354                            });355                        }356 357                        active_seqs.erase(s);358                        for (int i = 0; i < n_seq_dft; i++) {359                            if (i == s) {360                                continue;361                            }362                            if (drafts[i].active && drafts[i].tokens[i_dft] == drafts[s].tokens[i_dft]) {363                                // synchronize active status for sequences with the same drafted token364                                drafts[i].active = drafts[i].active && accept;365                                if (!drafts[i].active) {366                                    active_seqs.erase(s);367                                }368                            }369                        }370                    }371 372                    if (!accept) {373                        // all drafted tokens were rejected374                        // sample from the target model375                        LOG_DBG("all drafted tokens were rejected, sampling from residual distribution\n");376                        std::vector<float> probs(dist_tgt.size);377                        for (size_t i = 0; i < dist_tgt.size; ++i) {378                            probs[i] = dist_tgt.data[i].p;379                        }380 381                        std::discrete_distribution<> dist(probs.begin(), probs.end());382 383                        const int idx = dist(rng);384 385                        token_id = dist_tgt.data[idx].id;386                        common_sampler_accept(smpl, token_id, true);387                        token_str = common_token_to_piece(ctx_tgt, token_id);388                    }389                } else {390                    // greedy verification391 392                    // sample from the target model393                    LOG_DBG("sampling target: s_keep = %3d, i_dft = %3d, i_batch_tgt = %3d\n", s_keep, i_dft, drafts[s_keep].i_batch_tgt[i_dft]);394                    token_id = common_sampler_sample(smpl, ctx_tgt, drafts[s_keep].i_batch_tgt[i_dft]);395 396                    common_sampler_accept(smpl, token_id, true);397 398                    token_str = common_token_to_piece(ctx_tgt, token_id);399 400                    for (int s = 0; s < n_seq_dft; ++s) {401                        if (!drafts[s].active) {402                            continue;403                        }404 405                        if (i_dft < (int) drafts[s].tokens.size() && token_id == drafts[s].tokens[i_dft]) {406                            LOG_DBG("the sampled target token matches the %dth drafted token of sequence %d (%d, '%s') - accepted\n", i_dft, s, token_id, token_str.c_str());407 408                            s_keep = s;409                            accept = true;410                        } else {411                            drafts[s].active = false;412                        }413                    }414                }415 416                if (llama_vocab_is_eog(vocab_tgt, token_id)) {417                    has_eos = true;418                }419                ++n_predict;420 421                if (accept) {422                    ++n_accept;423                    ++n_past_tgt;424                    ++n_past_dft;425                    ++i_dft;426                    if (params.use_color) {427                        // Color token according to its origin sequence428                        LOG("\u001b[%dm%s\u001b[37m", (36 - s_keep % 6), token_str.c_str());429                    } else {430                        LOG("%s", token_str.c_str());431                    }432                    continue;433                } else {434                    LOG("%s", token_str.c_str());435                    break;436                }437            }438        }439 440        {441            LOG_DBG("the sampled target token (%d, '%s') did not match, or we ran out of drafted tokens\n", token_id, token_str.c_str());442 443            // TODO: simplify444            {445                LOG_DBG("keeping sequence %d, n_past_tgt = %d, n_past_dft = %d\n", s_keep, n_past_tgt, n_past_dft);446 447                llama_memory_seq_keep(mem_dft, s_keep);448                llama_memory_seq_cp  (mem_dft, s_keep, 0, -1, -1);449                llama_memory_seq_keep(mem_dft, 0);450 451                llama_memory_seq_rm  (mem_tgt, s_keep, n_past_tgt, -1);452                llama_memory_seq_keep(mem_tgt, s_keep);453                llama_memory_seq_cp  (mem_tgt, s_keep, 0, -1, -1);454                llama_memory_seq_keep(mem_tgt, 0);455            }456 457            for (int s = 0; s < n_seq_dft; ++s) {458                drafts[s].active = false;459                drafts[s].tokens.clear();460                drafts[s].i_batch_tgt.clear();461                drafts[s].dists.clear();462            }463            // note: will be erased after the speculation phase464            drafts[0].tokens.push_back(token_id);465            drafts[0].dists.push_back(std::vector<llama_token_data>());466            drafts[0].i_batch_tgt.push_back(0);467 468            common_batch_clear(batch_dft);469            common_batch_add  (batch_dft, token_id, n_past_dft, { 0 }, true);470 471            llama_memory_seq_rm(mem_dft, 0, n_past_dft, -1);472            // LOG_DBG("dft batch: %s\n", LOG_BATCH_TOSTR_PRETTY(ctx_dft, batch_dft).c_str());473            llama_decode(ctx_dft, batch_dft);474 475            ++n_past_dft;476        }477 478        if ((params.n_predict >= 0 && n_predict > params.n_predict) || has_eos) {479            break;480        }481 482        if (drafts[0].smpl) {483            common_sampler_free(drafts[0].smpl);484        }485        drafts[0].smpl = common_sampler_clone(smpl);486 487        int n_seq_cur  = 1;488        int n_past_cur = n_past_dft;489 490        for (int s = 0; s < n_seq_dft; ++s) {491            drafts[s].active   = false;492            drafts[s].drafting = false;493        }494        drafts[0].active      = true;495        drafts[0].drafting    = true;496        drafts[0].i_batch_dft = 0;497 498        common_batch_clear(batch_tgt);499        common_batch_add  (batch_tgt, drafts[0].tokens[0], n_past_tgt, { 0 }, true);500 501        // sample n_draft tokens from the draft model using tree-based sampling502        for (int i = 0; i < n_draft; ++i) {503            batch_dft.n_tokens = 0;504 505            for (int s = 0; s < n_seq_dft; ++s) {506                drafts[s].skip = false;507            }508 509            for (int s = 0; s < n_seq_dft; ++s) {510                if (!drafts[s].drafting || drafts[s].skip) {511                    continue;512                }513 514                common_sampler_sample(drafts[s].smpl, ctx_dft, drafts[s].i_batch_dft, true);515 516                const auto * cur_p = common_sampler_get_candidates(drafts[s].smpl, true);517 518                for (int k = 0; k < std::min(n_seq_dft + 3, (int) cur_p->size); ++k) {519                    LOG_DBG(" - draft candidate %3d for seq %3d, pos %3d: %6d (%8.3f) '%s'\n",520                            k, s, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str());521                }522 523                std::vector<int> sa(1, s);524 525                // attempt to split the branch if the probability is high enough526                for (int f = 1; f < 8; ++f) {527                    if (n_seq_cur < n_seq_dft && cur_p->data[f].p > p_draft_split) {528                        LOG_DBG("splitting seq %3d into %3d\n", s, n_seq_cur);529 530                        llama_memory_seq_rm(mem_dft,    n_seq_cur, -1, -1);531                        llama_memory_seq_cp(mem_dft, s, n_seq_cur, -1, -1);532 533                        // all previous tokens from this branch are now also part of the new branch534                        for (int t = 0; t < batch_tgt.n_tokens; ++t) {535                            for (int p = 0; p < batch_tgt.n_seq_id[t]; ++p) {536                                if (batch_tgt.seq_id[t][p] == s) {537                                    batch_tgt.seq_id[t][batch_tgt.n_seq_id[t]] = n_seq_cur;538                                    batch_tgt.n_seq_id[t]++;539                                    break;540                                }541                            }542                        }543 544                        // copy the draft state545                        drafts[n_seq_cur].active   = true;546                        drafts[n_seq_cur].drafting = true;547                        drafts[n_seq_cur].skip     = true;548 549                        drafts[n_seq_cur].tokens      = drafts[s].tokens;550                        drafts[n_seq_cur].dists       = drafts[s].dists;551                        drafts[n_seq_cur].i_batch_dft = drafts[s].i_batch_dft;552                        drafts[n_seq_cur].i_batch_tgt = drafts[s].i_batch_tgt;553 554                        if (drafts[n_seq_cur].smpl) {555                            common_sampler_free(drafts[n_seq_cur].smpl);556                        }557                        drafts[n_seq_cur].smpl = common_sampler_clone(drafts[s].smpl);558 559                        sa.push_back(n_seq_cur);560 561                        n_seq_cur++;562                    } else {563                        break;564                    }565                }566 567                // add drafted token for each sequence568                for (int is = 0; is < (int) sa.size(); ++is) {569                    const llama_token id = cur_p->data[is].id;570 571                    const int s = sa[is];572 573                    common_sampler_accept(drafts[s].smpl, id, true);574 575                    drafts[s].tokens.push_back(id);576                    // save cur_p.data into drafts[s].dists577                    drafts[s].dists.push_back({cur_p->data, cur_p->data + cur_p->size});578 579                    // add unique drafted tokens to the target batch580                    drafts[s].i_batch_tgt.push_back(batch_tgt.n_tokens);581 582                    common_batch_add(batch_tgt, id, n_past_tgt + i + 1, { s }, true);583 584                    // add the token to the batch for batched decoding with the draft model585                    drafts[s].i_batch_dft = batch_dft.n_tokens;586 587                    common_batch_add(batch_dft, id, n_past_cur, { s }, true);588 589                    if (batch_tgt.n_tokens > n_draft) {590                        drafts[s].drafting = false;591                    }592                }593            }594 595            // no sequence is drafting anymore596            if (batch_dft.n_tokens == 0) {597                break;598            }599 600            // evaluate the drafted tokens on the draft model601            llama_decode(ctx_dft, batch_dft);602            ++n_past_cur;603            ++n_drafted;604 605            if (batch_tgt.n_tokens > n_draft) {606                break;607            }608        }609 610        // evaluate the target model on the drafted tokens611        {612            llama_memory_seq_keep(mem_tgt, 0);613            for (int s = 1; s < n_seq_dft; ++s) {614                llama_memory_seq_cp(mem_tgt, 0, s, -1, -1);615            }616 617            // LOG_DBG("target batch: %s\n", LOG_BATCH_TOSTR_PRETTY(ctx_tgt, batch_tgt).c_str());618            llama_decode(ctx_tgt, batch_tgt);619            ++n_past_tgt;620        }621 622        // the first token is always proposed by the target model before the speculation loop so we erase it here623        for (int s = 0; s < n_seq_dft; ++s) {624            if (!drafts[s].active) {625                continue;626            }627 628            drafts[s].tokens.erase(drafts[s].tokens.begin());629            drafts[s].dists.erase(drafts[s].dists.begin());630        }631    }632 633    auto t_dec_end = ggml_time_us();634 635    LOG("\n\n");636 637    LOG_INF("encoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_input,   (t_enc_end - t_enc_start) / 1e6f, inp.size() / ((t_enc_end - t_enc_start) / 1e6f));638    LOG_INF("decoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_predict, (t_dec_end - t_dec_start) / 1e6f, n_predict  / ((t_dec_end - t_dec_start) / 1e6f));639 640    LOG_INF("\n");641    LOG_INF("n_draft   = %d\n", n_draft);642    LOG_INF("n_predict = %d\n", n_predict);643    LOG_INF("n_drafted = %d\n", n_drafted);644    LOG_INF("n_accept  = %d\n", n_accept);645    LOG_INF("accept    = %.3f%%\n", 100.0f * n_accept / n_drafted);646 647    LOG_INF("\n");648    LOG_INF("draft:\n\n");649    // TODO: print sampling/grammar timings for all drafts650    llama_perf_context_print(ctx_dft);651 652    LOG_INF("\n");653    LOG_INF("target:\n\n");654    common_perf_print(ctx_tgt, smpl);655 656    common_sampler_free(smpl);657    for (int s = 0; s < n_seq_dft; ++s) {658        common_sampler_free(drafts[s].smpl);659    }660 661    llama_batch_free(batch_dft);662 663    llama_backend_free();664 665    LOG("\n\n");666 667    return 0;668}669