KBaba7/llama.cpp
0
1#include <algorithm>2#include <array>3#include <cassert>4#include <chrono>5#include <cinttypes>6#include <clocale>7#include <cmath>8#include <cstdio>9#include <cstdlib>10#include <cstring>11#include <ctime>12#include <iterator>13#include <map>14#include <numeric>15#include <regex>16#include <sstream>17#include <string>18#include <thread>19#include <vector>20 21#include "common.h"22#include "ggml.h"23#include "llama.h"24 25#ifdef _WIN3226# define WIN32_LEAN_AND_MEAN27# ifndef NOMINMAX28# define NOMINMAX29# endif30# include <windows.h>31#endif32 33// utils34static uint64_t get_time_ns() {35 using clock = std::chrono::high_resolution_clock;36 return std::chrono::nanoseconds(clock::now().time_since_epoch()).count();37}38 39template <class T> static std::string join(const std::vector<T> & values, const std::string & delim) {40 std::ostringstream str;41 for (size_t i = 0; i < values.size(); i++) {42 str << values[i];43 if (i < values.size() - 1) {44 str << delim;45 }46 }47 return str.str();48}49 50template <typename T, typename F> static std::vector<std::string> transform_to_str(const std::vector<T> & values, F f) {51 std::vector<std::string> str_values;52 std::transform(values.begin(), values.end(), std::back_inserter(str_values), f);53 return str_values;54}55 56template <typename T> static T avg(const std::vector<T> & v) {57 if (v.empty()) {58 return 0;59 }60 T sum = std::accumulate(v.begin(), v.end(), T(0));61 return sum / (T) v.size();62}63 64template <typename T> static T stdev(const std::vector<T> & v) {65 if (v.size() <= 1) {66 return 0;67 }68 T mean = avg(v);69 T sq_sum = std::inner_product(v.begin(), v.end(), v.begin(), T(0));70 T stdev = std::sqrt(sq_sum / (T) (v.size() - 1) - mean * mean * (T) v.size() / (T) (v.size() - 1));71 return stdev;72}73 74static std::string get_cpu_info() {75 std::vector<std::string> cpu_list;76 for (size_t i = 0; i < ggml_backend_dev_count(); i++) {77 auto * dev = ggml_backend_dev_get(i);78 auto dev_type = ggml_backend_dev_type(dev);79 if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU || dev_type == GGML_BACKEND_DEVICE_TYPE_ACCEL) {80 cpu_list.push_back(ggml_backend_dev_description(dev));81 }82 }83 return join(cpu_list, ", ");84}85 86static std::string get_gpu_info() {87 std::vector<std::string> gpu_list;88 for (size_t i = 0; i < ggml_backend_dev_count(); i++) {89 auto * dev = ggml_backend_dev_get(i);90 auto dev_type = ggml_backend_dev_type(dev);91 if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU) {92 gpu_list.push_back(ggml_backend_dev_description(dev));93 }94 }95 return join(gpu_list, ", ");96}97 98// command line params99enum output_formats { NONE, CSV, JSON, JSONL, MARKDOWN, SQL };100 101static const char * output_format_str(output_formats format) {102 switch (format) {103 case NONE:104 return "none";105 case CSV:106 return "csv";107 case JSON:108 return "json";109 case JSONL:110 return "jsonl";111 case MARKDOWN:112 return "md";113 case SQL:114 return "sql";115 default:116 GGML_ABORT("invalid output format");117 }118}119 120static bool output_format_from_str(const std::string & s, output_formats & format) {121 if (s == "none") {122 format = NONE;123 } else if (s == "csv") {124 format = CSV;125 } else if (s == "json") {126 format = JSON;127 } else if (s == "jsonl") {128 format = JSONL;129 } else if (s == "md") {130 format = MARKDOWN;131 } else if (s == "sql") {132 format = SQL;133 } else {134 return false;135 }136 return true;137}138 139static const char * split_mode_str(llama_split_mode mode) {140 switch (mode) {141 case LLAMA_SPLIT_MODE_NONE:142 return "none";143 case LLAMA_SPLIT_MODE_LAYER:144 return "layer";145 case LLAMA_SPLIT_MODE_ROW:146 return "row";147 default:148 GGML_ABORT("invalid split mode");149 }150}151 152static std::string pair_str(const std::pair<int, int> & p) {153 static char buf[32];154 snprintf(buf, sizeof(buf), "%d,%d", p.first, p.second);155 return buf;156}157 158struct cmd_params {159 std::vector<std::string> model;160 std::vector<int> n_prompt;161 std::vector<int> n_gen;162 std::vector<std::pair<int, int>> n_pg;163 std::vector<int> n_batch;164 std::vector<int> n_ubatch;165 std::vector<ggml_type> type_k;166 std::vector<ggml_type> type_v;167 std::vector<int> n_threads;168 std::vector<std::string> cpu_mask;169 std::vector<bool> cpu_strict;170 std::vector<int> poll;171 std::vector<int> n_gpu_layers;172 std::vector<std::string> rpc_servers;173 std::vector<llama_split_mode> split_mode;174 std::vector<int> main_gpu;175 std::vector<bool> no_kv_offload;176 std::vector<bool> flash_attn;177 std::vector<std::vector<float>> tensor_split;178 std::vector<bool> use_mmap;179 std::vector<bool> embeddings;180 ggml_numa_strategy numa;181 int reps;182 ggml_sched_priority prio;183 int delay;184 bool verbose;185 bool progress;186 output_formats output_format;187 output_formats output_format_stderr;188};189 190static const cmd_params cmd_params_defaults = {191 /* model */ { "models/7B/ggml-model-q4_0.gguf" },192 /* n_prompt */ { 512 },193 /* n_gen */ { 128 },194 /* n_pg */ {},195 /* n_batch */ { 2048 },196 /* n_ubatch */ { 512 },197 /* type_k */ { GGML_TYPE_F16 },198 /* type_v */ { GGML_TYPE_F16 },199 /* n_threads */ { cpu_get_num_math() },200 /* cpu_mask */ { "0x0" },201 /* cpu_strict */ { false },202 /* poll */ { 50 },203 /* n_gpu_layers */ { 99 },204 /* rpc_servers */ { "" },205 /* split_mode */ { LLAMA_SPLIT_MODE_LAYER },206 /* main_gpu */ { 0 },207 /* no_kv_offload */ { false },208 /* flash_attn */ { false },209 /* tensor_split */ { std::vector<float>(llama_max_devices(), 0.0f) },210 /* use_mmap */ { true },211 /* embeddings */ { false },212 /* numa */ GGML_NUMA_STRATEGY_DISABLED,213 /* reps */ 5,214 /* prio */ GGML_SCHED_PRIO_NORMAL,215 /* delay */ 0,216 /* verbose */ false,217 /* progress */ false,218 /* output_format */ MARKDOWN,219 /* output_format_stderr */ NONE,220};221 222static void print_usage(int /* argc */, char ** argv) {223 printf("usage: %s [options]\n", argv[0]);224 printf("\n");225 printf("options:\n");226 printf(" -h, --help\n");227 printf(" -m, --model <filename> (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());228 printf(" -p, --n-prompt <n> (default: %s)\n",229 join(cmd_params_defaults.n_prompt, ",").c_str());230 printf(" -n, --n-gen <n> (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());231 printf(" -pg <pp,tg> (default: %s)\n",232 join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());233 printf(" -b, --batch-size <n> (default: %s)\n",234 join(cmd_params_defaults.n_batch, ",").c_str());235 printf(" -ub, --ubatch-size <n> (default: %s)\n",236 join(cmd_params_defaults.n_ubatch, ",").c_str());237 printf(" -ctk, --cache-type-k <t> (default: %s)\n",238 join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());239 printf(" -ctv, --cache-type-v <t> (default: %s)\n",240 join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());241 printf(" -t, --threads <n> (default: %s)\n",242 join(cmd_params_defaults.n_threads, ",").c_str());243 printf(" -C, --cpu-mask <hex,hex> (default: %s)\n",244 join(cmd_params_defaults.cpu_mask, ",").c_str());245 printf(" --cpu-strict <0|1> (default: %s)\n",246 join(cmd_params_defaults.cpu_strict, ",").c_str());247 printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());248 printf(" -ngl, --n-gpu-layers <n> (default: %s)\n",249 join(cmd_params_defaults.n_gpu_layers, ",").c_str());250 if (llama_supports_rpc()) {251 printf(" -rpc, --rpc <rpc_servers> (default: %s)\n",252 join(cmd_params_defaults.rpc_servers, ",").c_str());253 }254 printf(" -sm, --split-mode <none|layer|row> (default: %s)\n",255 join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());256 printf(" -mg, --main-gpu <i> (default: %s)\n",257 join(cmd_params_defaults.main_gpu, ",").c_str());258 printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n",259 join(cmd_params_defaults.no_kv_offload, ",").c_str());260 printf(" -fa, --flash-attn <0|1> (default: %s)\n",261 join(cmd_params_defaults.flash_attn, ",").c_str());262 printf(" -mmp, --mmap <0|1> (default: %s)\n",263 join(cmd_params_defaults.use_mmap, ",").c_str());264 printf(" --numa <distribute|isolate|numactl> (default: disabled)\n");265 printf(" -embd, --embeddings <0|1> (default: %s)\n",266 join(cmd_params_defaults.embeddings, ",").c_str());267 printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");268 printf(" -r, --repetitions <n> (default: %d)\n", cmd_params_defaults.reps);269 printf(" --prio <0|1|2|3> (default: %d)\n", cmd_params_defaults.prio);270 printf(" --delay <0...N> (seconds) (default: %d)\n", cmd_params_defaults.delay);271 printf(" -o, --output <csv|json|jsonl|md|sql> (default: %s)\n",272 output_format_str(cmd_params_defaults.output_format));273 printf(" -oe, --output-err <csv|json|jsonl|md|sql> (default: %s)\n",274 output_format_str(cmd_params_defaults.output_format_stderr));275 printf(" -v, --verbose (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0");276 printf(" --progress (default: %s)\n", cmd_params_defaults.progress ? "1" : "0");277 printf("\n");278 printf(279 "Multiple values can be given for each parameter by separating them with ',' or by specifying the parameter "280 "multiple times.\n");281}282 283static ggml_type ggml_type_from_name(const std::string & s) {284 if (s == "f16") {285 return GGML_TYPE_F16;286 }287 if (s == "bf16") {288 return GGML_TYPE_BF16;289 }290 if (s == "q8_0") {291 return GGML_TYPE_Q8_0;292 }293 if (s == "q4_0") {294 return GGML_TYPE_Q4_0;295 }296 if (s == "q4_1") {297 return GGML_TYPE_Q4_1;298 }299 if (s == "q5_0") {300 return GGML_TYPE_Q5_0;301 }302 if (s == "q5_1") {303 return GGML_TYPE_Q5_1;304 }305 if (s == "iq4_nl") {306 return GGML_TYPE_IQ4_NL;307 }308 309 return GGML_TYPE_COUNT;310}311 312static cmd_params parse_cmd_params(int argc, char ** argv) {313 cmd_params params;314 std::string arg;315 bool invalid_param = false;316 const std::string arg_prefix = "--";317 const char split_delim = ',';318 319 params.verbose = cmd_params_defaults.verbose;320 params.output_format = cmd_params_defaults.output_format;321 params.output_format_stderr = cmd_params_defaults.output_format_stderr;322 params.reps = cmd_params_defaults.reps;323 params.numa = cmd_params_defaults.numa;324 params.prio = cmd_params_defaults.prio;325 params.delay = cmd_params_defaults.delay;326 params.progress = cmd_params_defaults.progress;327 328 for (int i = 1; i < argc; i++) {329 arg = argv[i];330 if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {331 std::replace(arg.begin(), arg.end(), '_', '-');332 }333 334 if (arg == "-h" || arg == "--help") {335 print_usage(argc, argv);336 exit(0);337 } else if (arg == "-m" || arg == "--model") {338 if (++i >= argc) {339 invalid_param = true;340 break;341 }342 auto p = string_split<std::string>(argv[i], split_delim);343 params.model.insert(params.model.end(), p.begin(), p.end());344 } else if (arg == "-p" || arg == "--n-prompt") {345 if (++i >= argc) {346 invalid_param = true;347 break;348 }349 auto p = string_split<int>(argv[i], split_delim);350 params.n_prompt.insert(params.n_prompt.end(), p.begin(), p.end());351 } else if (arg == "-n" || arg == "--n-gen") {352 if (++i >= argc) {353 invalid_param = true;354 break;355 }356 auto p = string_split<int>(argv[i], split_delim);357 params.n_gen.insert(params.n_gen.end(), p.begin(), p.end());358 } else if (arg == "-pg") {359 if (++i >= argc) {360 invalid_param = true;361 break;362 }363 auto p = string_split<std::string>(argv[i], ',');364 if (p.size() != 2) {365 invalid_param = true;366 break;367 }368 params.n_pg.push_back({ std::stoi(p[0]), std::stoi(p[1]) });369 } else if (arg == "-b" || arg == "--batch-size") {370 if (++i >= argc) {371 invalid_param = true;372 break;373 }374 auto p = string_split<int>(argv[i], split_delim);375 params.n_batch.insert(params.n_batch.end(), p.begin(), p.end());376 } else if (arg == "-ub" || arg == "--ubatch-size") {377 if (++i >= argc) {378 invalid_param = true;379 break;380 }381 auto p = string_split<int>(argv[i], split_delim);382 params.n_ubatch.insert(params.n_ubatch.end(), p.begin(), p.end());383 } else if (arg == "-ctk" || arg == "--cache-type-k") {384 if (++i >= argc) {385 invalid_param = true;386 break;387 }388 auto p = string_split<std::string>(argv[i], split_delim);389 std::vector<ggml_type> types;390 for (const auto & t : p) {391 ggml_type gt = ggml_type_from_name(t);392 if (gt == GGML_TYPE_COUNT) {393 invalid_param = true;394 break;395 }396 types.push_back(gt);397 }398 if (invalid_param) {399 break;400 }401 params.type_k.insert(params.type_k.end(), types.begin(), types.end());402 } else if (arg == "-ctv" || arg == "--cache-type-v") {403 if (++i >= argc) {404 invalid_param = true;405 break;406 }407 auto p = string_split<std::string>(argv[i], split_delim);408 std::vector<ggml_type> types;409 for (const auto & t : p) {410 ggml_type gt = ggml_type_from_name(t);411 if (gt == GGML_TYPE_COUNT) {412 invalid_param = true;413 break;414 }415 types.push_back(gt);416 }417 if (invalid_param) {418 break;419 }420 params.type_v.insert(params.type_v.end(), types.begin(), types.end());421 } else if (arg == "-t" || arg == "--threads") {422 if (++i >= argc) {423 invalid_param = true;424 break;425 }426 auto p = string_split<int>(argv[i], split_delim);427 params.n_threads.insert(params.n_threads.end(), p.begin(), p.end());428 } else if (arg == "-C" || arg == "--cpu-mask") {429 if (++i >= argc) {430 invalid_param = true;431 break;432 }433 auto p = string_split<std::string>(argv[i], split_delim);434 params.cpu_mask.insert(params.cpu_mask.end(), p.begin(), p.end());435 } else if (arg == "--cpu-strict") {436 if (++i >= argc) {437 invalid_param = true;438 break;439 }440 auto p = string_split<bool>(argv[i], split_delim);441 params.cpu_strict.insert(params.cpu_strict.end(), p.begin(), p.end());442 } else if (arg == "--poll") {443 if (++i >= argc) {444 invalid_param = true;445 break;446 }447 auto p = string_split<int>(argv[i], split_delim);448 params.poll.insert(params.poll.end(), p.begin(), p.end());449 } else if (arg == "-ngl" || arg == "--n-gpu-layers") {450 if (++i >= argc) {451 invalid_param = true;452 break;453 }454 auto p = string_split<int>(argv[i], split_delim);455 params.n_gpu_layers.insert(params.n_gpu_layers.end(), p.begin(), p.end());456 } else if (llama_supports_rpc() && (arg == "-rpc" || arg == "--rpc")) {457 if (++i >= argc) {458 invalid_param = true;459 break;460 }461 params.rpc_servers.push_back(argv[i]);462 } else if (arg == "-sm" || arg == "--split-mode") {463 if (++i >= argc) {464 invalid_param = true;465 break;466 }467 auto p = string_split<std::string>(argv[i], split_delim);468 std::vector<llama_split_mode> modes;469 for (const auto & m : p) {470 llama_split_mode mode;471 if (m == "none") {472 mode = LLAMA_SPLIT_MODE_NONE;473 } else if (m == "layer") {474 mode = LLAMA_SPLIT_MODE_LAYER;475 } else if (m == "row") {476 mode = LLAMA_SPLIT_MODE_ROW;477 } else {478 invalid_param = true;479 break;480 }481 modes.push_back(mode);482 }483 if (invalid_param) {484 break;485 }486 params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end());487 } else if (arg == "-mg" || arg == "--main-gpu") {488 if (++i >= argc) {489 invalid_param = true;490 break;491 }492 params.main_gpu = string_split<int>(argv[i], split_delim);493 } else if (arg == "-nkvo" || arg == "--no-kv-offload") {494 if (++i >= argc) {495 invalid_param = true;496 break;497 }498 auto p = string_split<bool>(argv[i], split_delim);499 params.no_kv_offload.insert(params.no_kv_offload.end(), p.begin(), p.end());500 } else if (arg == "--numa") {501 if (++i >= argc) {502 invalid_param = true;503 break;504 } else {505 std::string value(argv[i]);506 /**/ if (value == "distribute" || value == "") {507 params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE;508 } else if (value == "isolate") {509 params.numa = GGML_NUMA_STRATEGY_ISOLATE;510 } else if (value == "numactl") {511 params.numa = GGML_NUMA_STRATEGY_NUMACTL;512 } else {513 invalid_param = true;514 break;515 }516 }517 } else if (arg == "-fa" || arg == "--flash-attn") {518 if (++i >= argc) {519 invalid_param = true;520 break;521 }522 auto p = string_split<bool>(argv[i], split_delim);523 params.flash_attn.insert(params.flash_attn.end(), p.begin(), p.end());524 } else if (arg == "-mmp" || arg == "--mmap") {525 if (++i >= argc) {526 invalid_param = true;527 break;528 }529 auto p = string_split<bool>(argv[i], split_delim);530 params.use_mmap.insert(params.use_mmap.end(), p.begin(), p.end());531 } else if (arg == "-embd" || arg == "--embeddings") {532 if (++i >= argc) {533 invalid_param = true;534 break;535 }536 auto p = string_split<bool>(argv[i], split_delim);537 params.embeddings.insert(params.embeddings.end(), p.begin(), p.end());538 } else if (arg == "-ts" || arg == "--tensor-split") {539 if (++i >= argc) {540 invalid_param = true;541 break;542 }543 for (auto ts : string_split<std::string>(argv[i], split_delim)) {544 // split string by ; and /545 const std::regex regex{ R"([;/]+)" };546 std::sregex_token_iterator it{ ts.begin(), ts.end(), regex, -1 };547 std::vector<std::string> split_arg{ it, {} };548 GGML_ASSERT(split_arg.size() <= llama_max_devices());549 550 std::vector<float> tensor_split(llama_max_devices());551 for (size_t i = 0; i < llama_max_devices(); ++i) {552 if (i < split_arg.size()) {553 tensor_split[i] = std::stof(split_arg[i]);554 } else {555 tensor_split[i] = 0.0f;556 }557 }558 params.tensor_split.push_back(tensor_split);559 }560 } else if (arg == "-r" || arg == "--repetitions") {561 if (++i >= argc) {562 invalid_param = true;563 break;564 }565 params.reps = std::stoi(argv[i]);566 } else if (arg == "--prio") {567 if (++i >= argc) {568 invalid_param = true;569 break;570 }571 params.prio = (enum ggml_sched_priority) std::stoi(argv[i]);572 } else if (arg == "--delay") {573 if (++i >= argc) {574 invalid_param = true;575 break;576 }577 params.delay = std::stoi(argv[i]);578 } else if (arg == "-o" || arg == "--output") {579 if (++i >= argc) {580 invalid_param = true;581 break;582 }583 invalid_param = !output_format_from_str(argv[i], params.output_format);584 } else if (arg == "-oe" || arg == "--output-err") {585 if (++i >= argc) {586 invalid_param = true;587 break;588 }589 invalid_param = !output_format_from_str(argv[i], params.output_format_stderr);590 } else if (arg == "-v" || arg == "--verbose") {591 params.verbose = true;592 } else if (arg == "--progress") {593 params.progress = true;594 } else {595 invalid_param = true;596 break;597 }598 }599 if (invalid_param) {600 fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());601 print_usage(argc, argv);602 exit(1);603 }604 605 // set defaults606 if (params.model.empty()) {607 params.model = cmd_params_defaults.model;608 }609 if (params.n_prompt.empty()) {610 params.n_prompt = cmd_params_defaults.n_prompt;611 }612 if (params.n_gen.empty()) {613 params.n_gen = cmd_params_defaults.n_gen;614 }615 if (params.n_pg.empty()) {616 params.n_pg = cmd_params_defaults.n_pg;617 }618 if (params.n_batch.empty()) {619 params.n_batch = cmd_params_defaults.n_batch;620 }621 if (params.n_ubatch.empty()) {622 params.n_ubatch = cmd_params_defaults.n_ubatch;623 }624 if (params.type_k.empty()) {625 params.type_k = cmd_params_defaults.type_k;626 }627 if (params.type_v.empty()) {628 params.type_v = cmd_params_defaults.type_v;629 }630 if (params.n_gpu_layers.empty()) {631 params.n_gpu_layers = cmd_params_defaults.n_gpu_layers;632 }633 if (params.rpc_servers.empty()) {634 params.rpc_servers = cmd_params_defaults.rpc_servers;635 }636 if (params.split_mode.empty()) {637 params.split_mode = cmd_params_defaults.split_mode;638 }639 if (params.main_gpu.empty()) {640 params.main_gpu = cmd_params_defaults.main_gpu;641 }642 if (params.no_kv_offload.empty()) {643 params.no_kv_offload = cmd_params_defaults.no_kv_offload;644 }645 if (params.flash_attn.empty()) {646 params.flash_attn = cmd_params_defaults.flash_attn;647 }648 if (params.tensor_split.empty()) {649 params.tensor_split = cmd_params_defaults.tensor_split;650 }651 if (params.use_mmap.empty()) {652 params.use_mmap = cmd_params_defaults.use_mmap;653 }654 if (params.embeddings.empty()) {655 params.embeddings = cmd_params_defaults.embeddings;656 }657 if (params.n_threads.empty()) {658 params.n_threads = cmd_params_defaults.n_threads;659 }660 if (params.cpu_mask.empty()) {661 params.cpu_mask = cmd_params_defaults.cpu_mask;662 }663 if (params.cpu_strict.empty()) {664 params.cpu_strict = cmd_params_defaults.cpu_strict;665 }666 if (params.poll.empty()) {667 params.poll = cmd_params_defaults.poll;668 }669 670 return params;671}672 673struct cmd_params_instance {674 std::string model;675 int n_prompt;676 int n_gen;677 int n_batch;678 int n_ubatch;679 ggml_type type_k;680 ggml_type type_v;681 int n_threads;682 std::string cpu_mask;683 bool cpu_strict;684 int poll;685 int n_gpu_layers;686 std::string rpc_servers_str;687 llama_split_mode split_mode;688 int main_gpu;689 bool no_kv_offload;690 bool flash_attn;691 std::vector<float> tensor_split;692 bool use_mmap;693 bool embeddings;694 695 llama_model_params to_llama_mparams() const {696 llama_model_params mparams = llama_model_default_params();697 698 mparams.n_gpu_layers = n_gpu_layers;699 if (!rpc_servers_str.empty()) {700 auto rpc_servers = string_split<std::string>(rpc_servers_str, ',');701 702 // add RPC devices703 if (!rpc_servers.empty()) {704 ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC");705 if (!rpc_reg) {706 fprintf(stderr, "%s: failed to find RPC backend\n", __func__);707 exit(1);708 }709 710 typedef ggml_backend_dev_t (*ggml_backend_rpc_add_device_t)(const char * endpoint);711 ggml_backend_rpc_add_device_t ggml_backend_rpc_add_device_fn = (ggml_backend_rpc_add_device_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_device");712 if (!ggml_backend_rpc_add_device_fn) {713 fprintf(stderr, "%s: failed to find RPC device add function\n", __func__);714 exit(1);715 }716 static std::vector<ggml_backend_dev_t> devices;717 devices.clear();718 for (const std::string & server : rpc_servers) {719 ggml_backend_dev_t dev = ggml_backend_rpc_add_device_fn(server.c_str());720 if (dev) {721 devices.push_back(dev);722 } else {723 fprintf(stderr, "%s: failed to add RPC device for server '%s'\n", __func__, server.c_str());724 exit(1);725 }726 }727 devices.push_back(nullptr);728 mparams.devices = devices.data();729 }730 }731 mparams.split_mode = split_mode;732 mparams.main_gpu = main_gpu;733 mparams.tensor_split = tensor_split.data();734 mparams.use_mmap = use_mmap;735 736 return mparams;737 }738 739 bool equal_mparams(const cmd_params_instance & other) const {740 return model == other.model && n_gpu_layers == other.n_gpu_layers && rpc_servers_str == other.rpc_servers_str &&741 split_mode == other.split_mode && main_gpu == other.main_gpu && use_mmap == other.use_mmap &&742 tensor_split == other.tensor_split;743 }744 745 llama_context_params to_llama_cparams() const {746 llama_context_params cparams = llama_context_default_params();747 748 cparams.n_ctx = n_prompt + n_gen;749 cparams.n_batch = n_batch;750 cparams.n_ubatch = n_ubatch;751 cparams.type_k = type_k;752 cparams.type_v = type_v;753 cparams.offload_kqv = !no_kv_offload;754 cparams.flash_attn = flash_attn;755 cparams.embeddings = embeddings;756 757 return cparams;758 }759};760 761static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_params & params) {762 std::vector<cmd_params_instance> instances;763 764 // this ordering minimizes the number of times that each model needs to be reloaded765 // clang-format off766 for (const auto & m : params.model)767 for (const auto & nl : params.n_gpu_layers)768 for (const auto & rpc : params.rpc_servers)769 for (const auto & sm : params.split_mode)770 for (const auto & mg : params.main_gpu)771 for (const auto & ts : params.tensor_split)772 for (const auto & mmp : params.use_mmap)773 for (const auto & embd : params.embeddings)774 for (const auto & nb : params.n_batch)775 for (const auto & nub : params.n_ubatch)776 for (const auto & tk : params.type_k)777 for (const auto & tv : params.type_v)778 for (const auto & nkvo : params.no_kv_offload)779 for (const auto & fa : params.flash_attn)780 for (const auto & nt : params.n_threads)781 for (const auto & cm : params.cpu_mask)782 for (const auto & cs : params.cpu_strict)783 for (const auto & pl : params.poll) {784 for (const auto & n_prompt : params.n_prompt) {785 if (n_prompt == 0) {786 continue;787 }788 cmd_params_instance instance = {789 /* .model = */ m,790 /* .n_prompt = */ n_prompt,791 /* .n_gen = */ 0,792 /* .n_batch = */ nb,793 /* .n_ubatch = */ nub,794 /* .type_k = */ tk,795 /* .type_v = */ tv,796 /* .n_threads = */ nt,797 /* .cpu_mask = */ cm,798 /* .cpu_strict = */ cs,799 /* .poll = */ pl,800 /* .n_gpu_layers = */ nl,801 /* .rpc_servers = */ rpc,802 /* .split_mode = */ sm,803 /* .main_gpu = */ mg,804 /* .no_kv_offload= */ nkvo,805 /* .flash_attn = */ fa,806 /* .tensor_split = */ ts,807 /* .use_mmap = */ mmp,808 /* .embeddings = */ embd,809 };810 instances.push_back(instance);811 }812 813 for (const auto & n_gen : params.n_gen) {814 if (n_gen == 0) {815 continue;816 }817 cmd_params_instance instance = {818 /* .model = */ m,819 /* .n_prompt = */ 0,820 /* .n_gen = */ n_gen,821 /* .n_batch = */ nb,822 /* .n_ubatch = */ nub,823 /* .type_k = */ tk,824 /* .type_v = */ tv,825 /* .n_threads = */ nt,826 /* .cpu_mask = */ cm,827 /* .cpu_strict = */ cs,828 /* .poll = */ pl,829 /* .n_gpu_layers = */ nl,830 /* .rpc_servers = */ rpc,831 /* .split_mode = */ sm,832 /* .main_gpu = */ mg,833 /* .no_kv_offload= */ nkvo,834 /* .flash_attn = */ fa,835 /* .tensor_split = */ ts,836 /* .use_mmap = */ mmp,837 /* .embeddings = */ embd,838 };839 instances.push_back(instance);840 }841 842 for (const auto & n_pg : params.n_pg) {843 if (n_pg.first == 0 && n_pg.second == 0) {844 continue;845 }846 cmd_params_instance instance = {847 /* .model = */ m,848 /* .n_prompt = */ n_pg.first,849 /* .n_gen = */ n_pg.second,850 /* .n_batch = */ nb,851 /* .n_ubatch = */ nub,852 /* .type_k = */ tk,853 /* .type_v = */ tv,854 /* .n_threads = */ nt,855 /* .cpu_mask = */ cm,856 /* .cpu_strict = */ cs,857 /* .poll = */ pl,858 /* .n_gpu_layers = */ nl,859 /* .rpc_servers = */ rpc,860 /* .split_mode = */ sm,861 /* .main_gpu = */ mg,862 /* .no_kv_offload= */ nkvo,863 /* .flash_attn = */ fa,864 /* .tensor_split = */ ts,865 /* .use_mmap = */ mmp,866 /* .embeddings = */ embd,867 };868 instances.push_back(instance);869 }870 }871 // clang-format on872 873 return instances;874}875 876struct test {877 static const std::string build_commit;878 static const int build_number;879 static const std::string cpu_info;880 static const std::string gpu_info;881 std::string model_filename;882 std::string model_type;883 uint64_t model_size;884 uint64_t model_n_params;885 int n_batch;886 int n_ubatch;887 int n_threads;888 std::string cpu_mask;889 bool cpu_strict;890 int poll;891 ggml_type type_k;892 ggml_type type_v;893 int n_gpu_layers;894 llama_split_mode split_mode;895 int main_gpu;896 bool no_kv_offload;897 bool flash_attn;898 std::vector<float> tensor_split;899 bool use_mmap;900 bool embeddings;901 int n_prompt;902 int n_gen;903 std::string test_time;904 std::vector<uint64_t> samples_ns;905 906 test(const cmd_params_instance & inst, const llama_model * lmodel, const llama_context * ctx) {907 model_filename = inst.model;908 char buf[128];909 llama_model_desc(lmodel, buf, sizeof(buf));910 model_type = buf;911 model_size = llama_model_size(lmodel);912 model_n_params = llama_model_n_params(lmodel);913 n_batch = inst.n_batch;914 n_ubatch = inst.n_ubatch;915 n_threads = inst.n_threads;916 cpu_mask = inst.cpu_mask;917 cpu_strict = inst.cpu_strict;918 poll = inst.poll;919 type_k = inst.type_k;920 type_v = inst.type_v;921 n_gpu_layers = inst.n_gpu_layers;922 split_mode = inst.split_mode;923 main_gpu = inst.main_gpu;924 no_kv_offload = inst.no_kv_offload;925 flash_attn = inst.flash_attn;926 tensor_split = inst.tensor_split;927 use_mmap = inst.use_mmap;928 embeddings = inst.embeddings;929 n_prompt = inst.n_prompt;930 n_gen = inst.n_gen;931 // RFC 3339 date-time format932 time_t t = time(NULL);933 std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));934 test_time = buf;935 936 (void) ctx;937 }938 939 uint64_t avg_ns() const { return ::avg(samples_ns); }940 941 uint64_t stdev_ns() const { return ::stdev(samples_ns); }942 943 std::vector<double> get_ts() const {944 int n_tokens = n_prompt + n_gen;945 std::vector<double> ts;946 std::transform(samples_ns.begin(), samples_ns.end(), std::back_inserter(ts),947 [n_tokens](uint64_t t) { return 1e9 * n_tokens / t; });948 return ts;949 }950 951 double avg_ts() const { return ::avg(get_ts()); }952 953 double stdev_ts() const { return ::stdev(get_ts()); }954 955 static std::string get_backend() {956 std::vector<std::string> backends;957 for (size_t i = 0; i < ggml_backend_reg_count(); i++) {958 auto * reg = ggml_backend_reg_get(i);959 std::string name = ggml_backend_reg_name(reg);960 if (name != "CPU") {961 backends.push_back(ggml_backend_reg_name(reg));962 }963 }964 return backends.empty() ? "CPU" : join(backends, ",");965 }966 967 static const std::vector<std::string> & get_fields() {968 static const std::vector<std::string> fields = {969 "build_commit", "build_number", "cpu_info", "gpu_info", "backends", "model_filename",970 "model_type", "model_size", "model_n_params", "n_batch", "n_ubatch", "n_threads",971 "cpu_mask", "cpu_strict", "poll", "type_k", "type_v", "n_gpu_layers",972 "split_mode", "main_gpu", "no_kv_offload", "flash_attn", "tensor_split", "use_mmap",973 "embeddings", "n_prompt", "n_gen", "test_time", "avg_ns", "stddev_ns",974 "avg_ts", "stddev_ts",975 };976 return fields;977 }978 979 enum field_type { STRING, BOOL, INT, FLOAT };980 981 static field_type get_field_type(const std::string & field) {982 if (field == "build_number" || field == "n_batch" || field == "n_ubatch" || field == "n_threads" ||983 field == "poll" || field == "model_size" || field == "model_n_params" || field == "n_gpu_layers" ||984 field == "main_gpu" || field == "n_prompt" || field == "n_gen" || field == "avg_ns" ||985 field == "stddev_ns") {986 return INT;987 }988 if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" || field == "flash_attn" ||989 field == "use_mmap" || field == "embeddings") {990 return BOOL;991 }992 if (field == "avg_ts" || field == "stddev_ts") {993 return FLOAT;994 }995 return STRING;996 }997 998 std::vector<std::string> get_values() const {999 std::string tensor_split_str;1000 int max_nonzero = 0;1001 for (size_t i = 0; i < llama_max_devices(); i++) {1002 if (tensor_split[i] > 0) {1003 max_nonzero = i;1004 }1005 }1006 for (int i = 0; i <= max_nonzero; i++) {1007 char buf[32];1008 snprintf(buf, sizeof(buf), "%.2f", tensor_split[i]);1009 tensor_split_str += buf;1010 if (i < max_nonzero) {1011 tensor_split_str += "/";1012 }1013 }1014 std::vector<std::string> values = { build_commit,1015 std::to_string(build_number),1016 cpu_info,1017 gpu_info,1018 get_backend(),1019 model_filename,1020 model_type,1021 std::to_string(model_size),1022 std::to_string(model_n_params),1023 std::to_string(n_batch),1024 std::to_string(n_ubatch),1025 std::to_string(n_threads),1026 cpu_mask,1027 std::to_string(cpu_strict),1028 std::to_string(poll),1029 ggml_type_name(type_k),1030 ggml_type_name(type_v),1031 std::to_string(n_gpu_layers),1032 split_mode_str(split_mode),1033 std::to_string(main_gpu),1034 std::to_string(no_kv_offload),1035 std::to_string(flash_attn),1036 tensor_split_str,1037 std::to_string(use_mmap),1038 std::to_string(embeddings),1039 std::to_string(n_prompt),1040 std::to_string(n_gen),1041 test_time,1042 std::to_string(avg_ns()),1043 std::to_string(stdev_ns()),1044 std::to_string(avg_ts()),1045 std::to_string(stdev_ts()) };1046 return values;1047 }1048 1049 std::map<std::string, std::string> get_map() const {1050 std::map<std::string, std::string> map;1051 auto fields = get_fields();1052 auto values = get_values();1053 std::transform(fields.begin(), fields.end(), values.begin(), std::inserter(map, map.end()),1054 std::make_pair<const std::string &, const std::string &>);1055 return map;1056 }1057};1058 1059const std::string test::build_commit = LLAMA_COMMIT;1060const int test::build_number = LLAMA_BUILD_NUMBER;1061const std::string test::cpu_info = get_cpu_info();1062const std::string test::gpu_info = get_gpu_info();1063 1064struct printer {1065 virtual ~printer() {}1066 1067 FILE * fout;1068 1069 virtual void print_header(const cmd_params & params) { (void) params; }1070 1071 virtual void print_test(const test & t) = 0;1072 1073 virtual void print_footer() {}1074};1075 1076struct csv_printer : public printer {1077 static std::string escape_csv(const std::string & field) {1078 std::string escaped = "\"";1079 for (auto c : field) {1080 if (c == '"') {1081 escaped += "\"";1082 }1083 escaped += c;1084 }1085 escaped += "\"";1086 return escaped;1087 }1088 1089 void print_header(const cmd_params & params) override {1090 std::vector<std::string> fields = test::get_fields();1091 fprintf(fout, "%s\n", join(fields, ",").c_str());1092 (void) params;1093 }1094 1095 void print_test(const test & t) override {1096 std::vector<std::string> values = t.get_values();1097 std::transform(values.begin(), values.end(), values.begin(), escape_csv);1098 fprintf(fout, "%s\n", join(values, ",").c_str());1099 }1100};1101 1102static std::string escape_json(const std::string & value) {1103 std::string escaped;1104 for (auto c : value) {1105 if (c == '"') {1106 escaped += "\\\"";1107 } else if (c == '\\') {1108 escaped += "\\\\";1109 } else if (c <= 0x1f) {1110 char buf[8];1111 snprintf(buf, sizeof(buf), "\\u%04x", c);1112 escaped += buf;1113 } else {1114 escaped += c;1115 }1116 }1117 return escaped;1118}1119 1120static std::string format_json_value(const std::string & field, const std::string & value) {1121 switch (test::get_field_type(field)) {1122 case test::STRING:1123 return "\"" + escape_json(value) + "\"";1124 case test::BOOL:1125 return value == "0" ? "false" : "true";1126 default:1127 return value;1128 }1129}1130 1131struct json_printer : public printer {1132 bool first = true;1133 1134 void print_header(const cmd_params & params) override {1135 fprintf(fout, "[\n");1136 (void) params;1137 }1138 1139 void print_fields(const std::vector<std::string> & fields, const std::vector<std::string> & values) {1140 assert(fields.size() == values.size());1141 for (size_t i = 0; i < fields.size(); i++) {1142 fprintf(fout, " \"%s\": %s,\n", fields.at(i).c_str(),1143 format_json_value(fields.at(i), values.at(i)).c_str());1144 }1145 }1146 1147 void print_test(const test & t) override {1148 if (first) {1149 first = false;1150 } else {1151 fprintf(fout, ",\n");1152 }1153 fprintf(fout, " {\n");1154 print_fields(test::get_fields(), t.get_values());1155 fprintf(fout, " \"samples_ns\": [ %s ],\n", join(t.samples_ns, ", ").c_str());1156 fprintf(fout, " \"samples_ts\": [ %s ]\n", join(t.get_ts(), ", ").c_str());1157 fprintf(fout, " }");1158 fflush(fout);1159 }1160 1161 void print_footer() override { fprintf(fout, "\n]\n"); }1162};1163 1164struct jsonl_printer : public printer {1165 void print_fields(const std::vector<std::string> & fields, const std::vector<std::string> & values) {1166 assert(fields.size() == values.size());1167 for (size_t i = 0; i < fields.size(); i++) {1168 fprintf(fout, "\"%s\": %s, ", fields.at(i).c_str(), format_json_value(fields.at(i), values.at(i)).c_str());1169 }1170 }1171 1172 void print_test(const test & t) override {1173 fprintf(fout, "{");1174 print_fields(test::get_fields(), t.get_values());1175 fprintf(fout, "\"samples_ns\": [ %s ],", join(t.samples_ns, ", ").c_str());1176 fprintf(fout, "\"samples_ts\": [ %s ]", join(t.get_ts(), ", ").c_str());1177 fprintf(fout, "}\n");1178 fflush(fout);1179 }1180};1181 1182struct markdown_printer : public printer {1183 std::vector<std::string> fields;1184 1185 static int get_field_width(const std::string & field) {1186 if (field == "model") {1187 return -30;1188 }1189 if (field == "t/s") {1190 return 20;1191 }1192 if (field == "size" || field == "params") {1193 return 10;1194 }1195 if (field == "n_gpu_layers") {1196 return 3;1197 }1198 if (field == "n_threads") {1199 return 7;1200 }