echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0479
1#include "arg.h"2#include "chat.h"3#include "common.h"4#include "llama.h"5#include "log.h"6 7#include <limits.h>8 9#include <algorithm>10#include <clocale>11#include <cmath>12#include <cstring>13#include <limits>14#include <random>15#include <string>16#include <vector>17 18enum diffusion_algorithm { ORIGIN = 0, ENTROPY_BASED = 1, MARGIN_BASED = 2, RANDOM = 3, CONFIDENCE_BASED = 4 };19 20// Unified transfer scheduling methods21enum transfer_schedule {22 TIMESTEP_BASED = 0, // Dream-style: (1.0 - s/t) * remaining23 BLOCK_BASED = 1, // LLaDA-style: process in blocks with get_num_transfer_tokens24};25 26typedef bool (*diffusion_step_callback_t)(int32_t step,27 int32_t total_steps,28 const llama_token * tokens,29 int32_t n_tokens,30 void * user_data);31 32struct diffusion_params {33 int32_t steps = 0;34 float temperature = 0;35 llama_token mask_token_id = LLAMA_TOKEN_NULL;36 diffusion_step_callback_t step_callback = nullptr;37 void * step_callback_user_data = nullptr;38 int32_t seed = 0;39 bool visual_mode = false;40 bool shift_logits = false; // Shift logits by -1 after decode41 42 float top_p = 0.;43 int32_t top_k = 0.;44 45 diffusion_algorithm algorithm = CONFIDENCE_BASED;46 transfer_schedule schedule = TIMESTEP_BASED;47 48 float cfg_scale = 0.; // Config scale for classifier-free guidance49 float eps = 0.; // Timestep scheduling50 int32_t block_length = 0; // Block size (for block scheduling)51 float alg_temp = 0; // algorithm temperature (0.0 = deterministic)52 bool add_gumbel_noise = false; // Add gumbel noise to the logits if temp > 0.053 54 int32_t max_length = 0; // Maximum sequence length55};56 57struct callback_data {58 diffusion_params * diff_params;59 const llama_vocab * vocab;60 int32_t n_input;61};62 63static float calculate_confidence(const llama_token_data_array & cur_p,64 diffusion_algorithm algorithm,65 std::mt19937 & rng) {66 switch (algorithm) {67 case CONFIDENCE_BASED:68 return cur_p.data[cur_p.selected].p; // Selected token probability69 70 case ENTROPY_BASED:71 {72 float entropy = 0.0f;73 const float epsilon = 1e-10f;74 for (size_t i = 0; i < cur_p.size; i++) {75 float prob = cur_p.data[i].p;76 entropy += prob * logf(prob + epsilon);77 }78 return -entropy; // Higher entropy = lower confidence79 }80 81 case MARGIN_BASED:82 return (cur_p.size > 1) ? cur_p.data[0].p - cur_p.data[1].p : cur_p.data[0].p;83 84 case RANDOM:85 {86 std::uniform_real_distribution<float> uniform(0.0f, 1.0f);87 return uniform(rng); // Random confidence88 }89 90 case ORIGIN:91 return cur_p.data[cur_p.selected].p;92 93 default:94 return 0.0f;95 }96}97 98// Unified transfer count calculation function99static int32_t calculate_transfer_count(int32_t step,100 int32_t total_steps,101 int32_t remaining_masked,102 transfer_schedule schedule,103 float eps,104 const std::vector<int32_t> & num_transfer_tokens = {}) {105 switch (schedule) {106 case TIMESTEP_BASED:107 {108 float t = 1.0f - (float) step / total_steps * (1.0f - eps);109 float s = 1.0f - (float) (step + 1) / total_steps * (1.0f - eps);110 float p_transfer = (step < total_steps - 1) ? (1.0f - s / t) : 1.0f;111 return (int32_t) (remaining_masked * p_transfer);112 }113 114 case BLOCK_BASED:115 if (!num_transfer_tokens.empty() && step < (int32_t) num_transfer_tokens.size()) {116 return num_transfer_tokens[step];117 }118 return remaining_masked / (total_steps - step); // Fallback119 120 default:121 return remaining_masked / (total_steps - step);122 }123}124 125static bool diffusion_step_callback(int32_t step,126 int32_t total_steps,127 const llama_token * tokens,128 int32_t n_tokens,129 void * user_data) {130 (void) user_data;131 132 callback_data * data = static_cast<callback_data *>(user_data);133 134 auto print_progress_bar = [](int32_t step, int32_t total_steps) {135 int progress_percent = (step * 100) / total_steps;136 int progress_bars = (step * 50) / total_steps;137 LOG_INF("\rdiffusion step: %d/%d [%s%s] %d%%",138 step,139 total_steps,140 std::string(progress_bars, '=').c_str(),141 std::string(50 - progress_bars, ' ').c_str(),142 progress_percent);143 };144 145 if (data->diff_params->visual_mode) {146 // Visual mode: clear147 LOG_INF("\033[2J\033[H"); // Clear screen and move cursor to top-left148 149 print_progress_bar(step, total_steps);150 151 LOG_INF("\n");152 153 std::string current_text = " ";154 155 for (int32_t i = data->n_input; i < n_tokens; i++) {156 std::string token_str;157 if (tokens[i] != llama_vocab_mask(data->vocab)) {158 char piece[256];159 int n_chars = llama_token_to_piece(data->vocab, tokens[i], piece, sizeof(piece), 0, false);160 if (n_chars > 0) {161 piece[n_chars] = '\0';162 token_str = piece;163 }164 } else {165 token_str = " ";166 }167 168 current_text += token_str;169 }170 171 LOG_INF("%s\n", current_text.c_str());172 } else {173 print_progress_bar(step, total_steps);174 }175 176 return true;177}178 179static void add_gumbel_noise(float * logits, int32_t n_vocab, float temperature, std::mt19937 & rng) {180 if (temperature == 0.0f) {181 return;182 }183 184 std::uniform_real_distribution<double> uniform(0.0, 1.0);185 for (int32_t i = 0; i < n_vocab; i++) {186 double noise = uniform(rng);187 // Prevent log(0)188 noise = std::max(noise, 1e-20);189 double gumbel_noise = std::pow(-std::log(noise), temperature);190 logits[i] = std::exp(logits[i]) / gumbel_noise;191 }192}193 194static std::vector<int32_t> get_num_transfer_tokens(int32_t mask_count, int32_t steps) {195 std::vector<int32_t> num_transfer_tokens(steps);196 197 int32_t base = mask_count / steps;198 int32_t remainder = mask_count % steps;199 200 for (int32_t i = 0; i < steps; i++) {201 num_transfer_tokens[i] = base + (i < remainder ? 1 : 0);202 }203 204 return num_transfer_tokens;205}206 207static void diffusion_generate(llama_context * ctx,208 const llama_token * input_tokens,209 llama_token * output_tokens,210 int32_t n_input,211 const diffusion_params & params,212 int32_t & n_generated) {213 n_generated = 0;214 if (!ctx || !input_tokens || !output_tokens || n_input <= 0 || params.max_length <= n_input) {215 return;216 }217 218 const llama_model * model = llama_get_model(ctx);219 220 // Initialize with input and pad with mask tokens221 std::copy(input_tokens, input_tokens + n_input, output_tokens);222 std::fill(output_tokens + n_input, output_tokens + params.max_length, params.mask_token_id);223 224 std::mt19937 rng(params.seed);225 226 llama_set_causal_attn(ctx, false);227 228 int32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));229 230 std::vector<llama_token_data> candidates(n_vocab);231 std::vector<llama_token_data> conf_candidates;232 conf_candidates.reserve(params.max_length);233 std::vector<int32_t> mask_positions;234 mask_positions.reserve(params.max_length);235 236 // Setup sampler chain237 struct llama_sampler * sampler = llama_sampler_chain_init(llama_sampler_chain_default_params());238 if (params.top_k > 0) {239 llama_sampler_chain_add(sampler, llama_sampler_init_top_k(params.top_k));240 }241 if (params.top_p < 1.0f) {242 llama_sampler_chain_add(sampler, llama_sampler_init_top_p(params.top_p, 1));243 }244 if (params.temperature > 0.0f) {245 llama_sampler_chain_add(sampler, llama_sampler_init_temp(params.temperature));246 }247 llama_sampler_chain_add(sampler, llama_sampler_init_dist(params.seed));248 249 struct llama_sampler * dist_sampler = llama_sampler_init_dist(params.seed);250 251 llama_batch batch = llama_batch_init(params.max_length, 0, 1);252 batch.n_tokens = params.max_length;253 254 // Pre-allocate buffers for CFG if needed255 int32_t logits_size = n_vocab * params.max_length;256 std::vector<float> cond_logits_buffer;257 std::vector<llama_token> un_x_buffer;258 if (params.cfg_scale > 0.0f) {259 cond_logits_buffer.resize(logits_size);260 un_x_buffer.resize(params.max_length);261 }262 263 // For block-based processing264 std::vector<int32_t> num_transfer_tokens;265 int32_t num_blocks = 1;266 int32_t steps_per_block = params.steps;267 268 if (params.schedule == BLOCK_BASED) {269 GGML_ASSERT(params.max_length % params.block_length == 0);270 num_blocks = params.max_length / params.block_length;271 GGML_ASSERT(params.steps % num_blocks == 0);272 steps_per_block = params.steps / num_blocks;273 }274 275 std::vector<float> confidence(params.max_length);276 277 int64_t total_sampling_time = 0;278 int64_t total_time = 0;279 int64_t time_start = ggml_time_us();280 281 for (int block_num = 0; block_num < num_blocks; block_num++) {282 int32_t block_start = (params.schedule == BLOCK_BASED) ? n_input + block_num * params.block_length : 0;283 int32_t block_end = (params.schedule == BLOCK_BASED) ?284 std::min(n_input + (block_num + 1) * params.block_length, params.max_length) :285 params.max_length;286 287 // Count masked tokens in current block for block-based processing288 if (params.schedule == BLOCK_BASED) {289 int32_t block_mask_count = 0;290 for (int i = block_start; i < block_end; i++) {291 if (output_tokens[i] == params.mask_token_id) {292 block_mask_count++;293 }294 }295 num_transfer_tokens = get_num_transfer_tokens(block_mask_count, steps_per_block);296 }297 298 for (int32_t step = 0; step < steps_per_block; step++) {299 int32_t global_step = block_num * steps_per_block + step;300 301 if (params.step_callback) {302 if (!params.step_callback(303 global_step, params.steps, output_tokens, params.max_length, params.step_callback_user_data)) {304 break;305 }306 }307 308 // Setup batch309 for (int32_t i = 0; i < params.max_length; i++) {310 batch.token[i] = output_tokens[i];311 batch.pos[i] = i;312 batch.n_seq_id[i] = 1;313 batch.seq_id[i][0] = 0;314 batch.logits[i] = 1;315 }316 317 float * logits = nullptr;318 319 if (params.cfg_scale > 0.0f) {320 int ret = llama_decode(ctx, batch);321 if (ret != 0) {322 LOG_ERR("Failed to generate conditional");323 break;324 }325 float * cond_logits_ptr = llama_get_logits(ctx);326 std::memcpy(cond_logits_buffer.data(), cond_logits_ptr, logits_size * sizeof(float));327 328 // Unconditional generation (mask input)329 std::copy(output_tokens, output_tokens + params.max_length, un_x_buffer.begin());330 for (int32_t i = 0; i < n_input; i++) {331 un_x_buffer[i] = params.mask_token_id;332 }333 334 for (int32_t i = 0; i < params.max_length; i++) {335 batch.token[i] = un_x_buffer[i];336 }337 ret = llama_decode(ctx, batch);338 if (ret != 0) {339 LOG_ERR("Failed to generate unconditional");340 break;341 }342 float * uncond_logits = llama_get_logits(ctx);343 344 // Apply CFG345 for (int32_t i = 0; i < logits_size; i++) {346 cond_logits_buffer[i] =347 uncond_logits[i] + (params.cfg_scale + 1.0f) * (cond_logits_buffer[i] - uncond_logits[i]);348 }349 logits = cond_logits_buffer.data();350 } else {351 int ret = llama_decode(ctx, batch);352 if (ret != 0) {353 LOG_ERR("%s: failed to decode at step %d, ret = %d\n", __func__, global_step, ret);354 break;355 }356 logits = llama_get_logits(ctx);357 }358 359 if (!logits) {360 LOG_ERR("%s: failed to get logits at step %d\n", __func__, global_step);361 break;362 }363 364 auto get_logits_for_pos = [&](int32_t pos) -> const float * {365 if (params.shift_logits) {366 return pos == 0 ? logits : logits + (pos - 1) * n_vocab;367 }368 return logits + (pos) *n_vocab;369 };370 371 int64_t time_start_sampling = ggml_time_us();372 373 mask_positions.clear();374 for (int32_t i = 0; i < params.max_length; i++) {375 if (output_tokens[i] == params.mask_token_id) {376 // For block-based, only consider current block377 if (params.schedule != BLOCK_BASED || (i >= block_start && i < block_end)) {378 mask_positions.push_back(i);379 }380 }381 }382 383 if (mask_positions.empty()) {384 break;385 }386 387 if (params.add_gumbel_noise && params.temperature > 0.0f) {388 add_gumbel_noise(logits, n_vocab, params.temperature, rng);389 }390 391 if (params.algorithm == ORIGIN) {392 int32_t transfer_count = calculate_transfer_count(393 step, steps_per_block, mask_positions.size(), params.schedule, params.eps, num_transfer_tokens);394 float p_transfer = (float) transfer_count / mask_positions.size();395 396 for (int32_t pos : mask_positions) {397 if (std::uniform_real_distribution<float>(0.0f, 1.0f)(rng) < p_transfer) {398 const float * pos_logits = get_logits_for_pos(pos);399 for (int32_t token_id = 0; token_id < n_vocab; token_id++) {400 candidates[token_id].id = token_id;401 candidates[token_id].logit = pos_logits[token_id];402 candidates[token_id].p = 0.0f;403 }404 405 llama_token_data_array cur_p = {406 candidates.data(),407 (size_t) n_vocab,408 -1,409 false,410 };411 412 llama_sampler_apply(sampler, &cur_p);413 output_tokens[pos] = cur_p.data[cur_p.selected].id;414 }415 }416 } else {417 std::vector<std::pair<float, int32_t>> confidences;418 std::vector<llama_token> sampled_tokens(mask_positions.size());419 420 for (size_t i = 0; i < mask_positions.size(); i++) {421 int32_t pos = mask_positions[i];422 const float * pos_logits = get_logits_for_pos(pos);423 424 for (int32_t token_id = 0; token_id < n_vocab; token_id++) {425 candidates[token_id].logit = pos_logits[token_id];426 candidates[token_id].p = 0.0f;427 candidates[token_id].id = token_id;428 }429 430 llama_token_data_array cur_p = {431 candidates.data(),432 candidates.size(),433 -1,434 false,435 };436 437 llama_sampler_apply(sampler, &cur_p);438 llama_token sampled_token = cur_p.data[cur_p.selected].id;439 440 float conf = calculate_confidence(cur_p, params.algorithm, rng);441 442 sampled_tokens[i] = sampled_token;443 confidences.emplace_back(conf, i);444 }445 446 int32_t transfer_count = calculate_transfer_count(447 step, steps_per_block, mask_positions.size(), params.schedule, params.eps, num_transfer_tokens);448 449 if (transfer_count > 0) {450 if (params.alg_temp == 0.0f) {451 std::partial_sort(confidences.begin(),452 confidences.begin() + std::min(transfer_count, (int32_t) confidences.size()),453 confidences.end(),454 [](const std::pair<float, int32_t> & a, const std::pair<float, int32_t> & b) {455 if (a.first != b.first) {456 return a.first > b.first;457 }458 return a.second < b.second;459 });460 461 for (int32_t i = 0; i < std::min(transfer_count, (int32_t) confidences.size()); i++) {462 int32_t mask_idx = confidences[i].second;463 int32_t pos = mask_positions[mask_idx];464 output_tokens[pos] = sampled_tokens[mask_idx];465 }466 } else {467 conf_candidates.clear();468 for (size_t i = 0; i < confidences.size(); i++) {469 float conf_logit = confidences[i].first / params.alg_temp;470 conf_candidates.emplace_back(llama_token_data{ (int32_t) i, conf_logit, 0.0f });471 }472 473 llama_token_data_array conf_array = {474 conf_candidates.data(),475 conf_candidates.size(),476 -1,477 false,478 };479 480 for (int32_t i = 0; i < std::min(transfer_count, (int32_t) confidences.size()); i++) {481 llama_sampler_apply(dist_sampler, &conf_array);482 int32_t selected_idx = conf_array.selected;483 int32_t mask_idx = selected_idx;484 int32_t pos = mask_positions[mask_idx];485 output_tokens[pos] = sampled_tokens[mask_idx];486 487 conf_candidates[selected_idx].p = 0.0f;488 conf_array.selected = -1;489 }490 }491 }492 }493 494 int64_t time_end_sampling = ggml_time_us();495 total_sampling_time += time_end_sampling - time_start_sampling;496 }497 }498 499 int64_t time_end = ggml_time_us();500 total_time += time_end - time_start;501 502 LOG_INF("\ntotal time: %0.2fms, time per step: %0.2fms, sampling time per step: %0.2fms\n",503 total_time / 1000.0,504 total_time / 1000.0 / params.steps,505 total_sampling_time / 1000.0 / params.steps);506 507 llama_batch_free(batch);508 llama_sampler_free(sampler);509 llama_sampler_free(dist_sampler);510 511 n_generated = params.max_length;512}513 514static std::string format_input_text(const std::string & prompt, const std::string & system_prompt, bool use_chat_template, llama_model * model) {515 if (!use_chat_template) {516 return prompt;517 }518 519 auto chat_templates = common_chat_templates_init(model, "");520 common_chat_templates_inputs inputs;521 common_chat_msg system_msg;522 523 if (!system_prompt.empty()) {524 system_msg.role = "system";525 system_msg.content = system_prompt;526 inputs.messages.push_back(system_msg);527 }528 529 common_chat_msg user_msg;530 user_msg.role = "user";531 user_msg.content = prompt;532 533 inputs.messages.push_back(user_msg);534 inputs.add_generation_prompt = true;535 536 auto result = common_chat_templates_apply(chat_templates.get(), inputs);537 538 return result.prompt;539}540 541int main(int argc, char ** argv) {542 std::setlocale(LC_NUMERIC, "C");543 544 ggml_time_init();545 546 common_params params;547 548 common_init();549 550 if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_DIFFUSION)) {551 return 1;552 }553 554 llama_backend_init();555 556 llama_model_params model_params = llama_model_default_params();557 model_params.n_gpu_layers = params.n_gpu_layers;558 model_params.devices = params.devices.data();559 model_params.use_mmap = params.use_mmap;560 model_params.use_direct_io = params.use_direct_io;561 model_params.use_mlock = params.use_mlock;562 model_params.check_tensors = params.check_tensors;563 564 llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);565 if (!model) {566 LOG_ERR("error: failed to load model '%s'\n", params.model.path.c_str());567 return 1;568 }569 570 if (!llama_model_is_diffusion(model)) {571 LOG_ERR("error: unsupported model for diffusion");572 llama_model_free(model);573 return 1;574 }575 576 llama_context_params ctx_params = llama_context_default_params();577 ctx_params.n_ctx = params.n_ctx;578 ctx_params.n_batch = params.n_batch;579 ctx_params.n_ubatch = params.n_ubatch;580 ctx_params.flash_attn_type = params.flash_attn_type;581 ctx_params.no_perf = params.no_perf;582 ctx_params.type_k = params.cache_type_k;583 ctx_params.type_v = params.cache_type_v;584 585 llama_context * ctx = llama_init_from_model(model, ctx_params);586 if (!ctx) {587 LOG_ERR("error: failed to create context\n");588 llama_model_free(model);589 return 1;590 }591 592 llama_set_n_threads(ctx, params.cpuparams.n_threads, params.cpuparams_batch.n_threads);593 594 const llama_vocab * vocab = llama_model_get_vocab(model);595 596 std::string formatted_prompt = format_input_text(params.prompt, params.system_prompt, params.enable_chat_template, model);597 598 std::vector<llama_token> input_tokens = common_tokenize(vocab,599 formatted_prompt,600 /*add special tokens*/ true,601 /*parse special*/ true);602 603 int n_input = input_tokens.size();604 605 if (static_cast<uint32_t>(n_input) >= llama_n_ctx(ctx)) {606 LOG_ERR("error: input too long (%d tokens), max context is %d\n", n_input, llama_n_ctx(ctx));607 llama_free(ctx);608 llama_model_free(model);609 return 1;610 }611 612 llama_token mask_token_id = llama_vocab_mask(vocab);613 614 GGML_ASSERT(mask_token_id != LLAMA_TOKEN_NULL);615 616 bool visual_mode = params.diffusion.visual_mode;617 618 int32_t n_generated = 0;619 std::vector<llama_token> output_tokens(params.n_ubatch);620 621 struct diffusion_params diff_params;622 623 char shift_logits_str[8];624 if (llama_model_meta_val_str(model, "diffusion.shift_logits", shift_logits_str, sizeof(shift_logits_str)) >= 0) {625 diff_params.shift_logits = (strcmp(shift_logits_str, "true") == 0);626 } else {627 diff_params.shift_logits = true;628 }629 630 //Use either eps or block length, but not both631 GGML_ASSERT((params.diffusion.eps == 0) ^ (params.diffusion.block_length == 0));632 633 if (params.diffusion.eps) {634 diff_params.schedule = TIMESTEP_BASED;635 diff_params.eps = params.diffusion.eps;636 } else if (params.diffusion.block_length) {637 diff_params.schedule = BLOCK_BASED;638 diff_params.block_length = params.diffusion.block_length;639 }640 641 diff_params.mask_token_id = mask_token_id;642 diff_params.seed = params.sampling.seed;643 diff_params.temperature = params.sampling.temp;644 diff_params.steps = params.diffusion.steps;645 diff_params.algorithm = static_cast<diffusion_algorithm>(params.diffusion.algorithm);646 diff_params.max_length = params.n_ubatch;647 diff_params.top_p = params.sampling.top_p;648 diff_params.top_k = params.sampling.top_k;649 diff_params.visual_mode = params.diffusion.visual_mode;650 diff_params.add_gumbel_noise = params.diffusion.add_gumbel_noise;651 652 diff_params.step_callback = diffusion_step_callback;653 callback_data cb_data = { &diff_params, vocab, n_input };654 diff_params.step_callback_user_data = &cb_data;655 656 const char * alg_names[] = { "ORIGIN", "ENTROPY_BASED", "MARGIN_BASED", "RANDOM", "CONFIDENCE_BASED" };657 const char * sched_names[] = { "TIMESTEP_BASED", "BLOCK_BASED" };658 const char * alg_name =659 (diff_params.algorithm >= 0 && diff_params.algorithm <= 4) ? alg_names[diff_params.algorithm] : "UNKNOWN";660 const char * sched_name =661 (diff_params.schedule >= 0 && diff_params.schedule <= 1) ? sched_names[diff_params.schedule] : "UNKNOWN";662 663 LOG_INF("diffusion_params: - %-25s llama_token = %d\n", "mask_token_id", mask_token_id);664 LOG_INF("diffusion_params: - %-25s u32 = %d\n", "steps", diff_params.steps);665 LOG_INF("diffusion_params: - %-25s u32 = %d\n", "max_length", diff_params.max_length);666 LOG_INF("diffusion_params: - %-25s enum = %d (%s)\n", "algorithm", diff_params.algorithm, alg_name);667 LOG_INF("diffusion_params: - %-25s enum = %d (%s)\n", "schedule", diff_params.schedule, sched_name);668 LOG_INF("diffusion_params: - %-25s f32 = %.3f\n", "temperature", diff_params.temperature);669 if (diff_params.schedule == TIMESTEP_BASED) {670 LOG_INF("diffusion_params: - %-25s f32 = %.6f\n", "eps", diff_params.eps);671 LOG_INF("diffusion_params: - %-25s f32 = %.3f\n", "alg_temp", diff_params.alg_temp);672 }673 if (diff_params.schedule == BLOCK_BASED) {674 LOG_INF("diffusion_params: - %-25s u32 = %d\n", "block_length", diff_params.block_length);675 LOG_INF("diffusion_params: - %-25s f32 = %.3f\n", "cfg_scale", diff_params.cfg_scale);676 }677 678 diffusion_generate(ctx, input_tokens.data(), output_tokens.data(), n_input, diff_params, n_generated);679 680 if (n_generated > 0) {681 if (visual_mode) {682 //clear screen and move cursor to top-left683 LOG_INF("\033[2J\033[H");684 }685 686 output_tokens.erase(output_tokens.begin(), output_tokens.begin() + n_input);687 std::string output_data = common_detokenize(vocab, output_tokens, false);688 LOG_INF("\n%s\n", output_data.c_str());689 } else {690 LOG_INF("Error: diffusion generation failed\n");691 }692 693 llama_free(ctx);694 llama_model_free(model);695 llama_backend_free();696 697 return 0;698}699