KBaba7/llama.cpp
0
1#include "arg.h"2 3#include "log.h"4#include "sampling.h"5 6#include <algorithm>7#include <climits>8#include <cstdarg>9#include <fstream>10#include <regex>11#include <set>12#include <string>13#include <thread>14#include <vector>15 16#include "json-schema-to-grammar.h"17 18using json = nlohmann::ordered_json;19 20common_arg & common_arg::set_examples(std::initializer_list<enum llama_example> examples) {21 this->examples = std::move(examples);22 return *this;23}24 25common_arg & common_arg::set_excludes(std::initializer_list<enum llama_example> excludes) {26 this->excludes = std::move(excludes);27 return *this;28}29 30common_arg & common_arg::set_env(const char * env) {31 help = help + "\n(env: " + env + ")";32 this->env = env;33 return *this;34}35 36common_arg & common_arg::set_sparam() {37 is_sparam = true;38 return *this;39}40 41bool common_arg::in_example(enum llama_example ex) {42 return examples.find(ex) != examples.end();43}44 45bool common_arg::is_exclude(enum llama_example ex) {46 return excludes.find(ex) != excludes.end();47}48 49bool common_arg::get_value_from_env(std::string & output) {50 if (env == nullptr) return false;51 char * value = std::getenv(env);52 if (value) {53 output = value;54 return true;55 }56 return false;57}58 59bool common_arg::has_value_from_env() {60 return env != nullptr && std::getenv(env);61}62 63static std::vector<std::string> break_str_into_lines(std::string input, size_t max_char_per_line) {64 std::vector<std::string> result;65 std::istringstream iss(input);66 std::string line;67 auto add_line = [&](const std::string& l) {68 if (l.length() <= max_char_per_line) {69 result.push_back(l);70 } else {71 std::istringstream line_stream(l);72 std::string word, current_line;73 while (line_stream >> word) {74 if (current_line.length() + !current_line.empty() + word.length() > max_char_per_line) {75 if (!current_line.empty()) result.push_back(current_line);76 current_line = word;77 } else {78 current_line += (!current_line.empty() ? " " : "") + word;79 }80 }81 if (!current_line.empty()) result.push_back(current_line);82 }83 };84 while (std::getline(iss, line)) {85 add_line(line);86 }87 return result;88}89 90std::string common_arg::to_string() {91 // params for printing to console92 const static int n_leading_spaces = 40;93 const static int n_char_per_line_help = 70; // TODO: detect this based on current console94 std::string leading_spaces(n_leading_spaces, ' ');95 96 std::ostringstream ss;97 for (const auto arg : args) {98 if (arg == args.front()) {99 if (args.size() == 1) {100 ss << arg;101 } else {102 // first arg is usually abbreviation, we need padding to make it more beautiful103 auto tmp = std::string(arg) + ", ";104 auto spaces = std::string(std::max(0, 7 - (int)tmp.size()), ' ');105 ss << tmp << spaces;106 }107 } else {108 ss << arg << (arg != args.back() ? ", " : "");109 }110 }111 if (value_hint) ss << " " << value_hint;112 if (value_hint_2) ss << " " << value_hint_2;113 if (ss.tellp() > n_leading_spaces - 3) {114 // current line is too long, add new line115 ss << "\n" << leading_spaces;116 } else {117 // padding between arg and help, same line118 ss << std::string(leading_spaces.size() - ss.tellp(), ' ');119 }120 const auto help_lines = break_str_into_lines(help, n_char_per_line_help);121 for (const auto & line : help_lines) {122 ss << (&line == &help_lines.front() ? "" : leading_spaces) << line << "\n";123 }124 return ss.str();125}126 127//128// utils129//130 131static void common_params_handle_model_default(132 std::string & model,133 const std::string & model_url,134 std::string & hf_repo,135 std::string & hf_file,136 const std::string & hf_token,137 const std::string & model_default) {138 if (!hf_repo.empty()) {139 // short-hand to avoid specifying --hf-file -> default it to --model140 if (hf_file.empty()) {141 if (model.empty()) {142 auto auto_detected = common_get_hf_file(hf_repo, hf_token);143 if (auto_detected.first.empty() || auto_detected.second.empty()) {144 exit(1); // built without CURL, error message already printed145 }146 hf_repo = auto_detected.first;147 hf_file = auto_detected.second;148 } else {149 hf_file = model;150 }151 }152 // make sure model path is present (for caching purposes)153 if (model.empty()) {154 // this is to avoid different repo having same file name, or same file name in different subdirs155 std::string filename = hf_repo + "_" + hf_file;156 // to make sure we don't have any slashes in the filename157 string_replace_all(filename, "/", "_");158 model = fs_get_cache_file(filename);159 }160 } else if (!model_url.empty()) {161 if (model.empty()) {162 auto f = string_split<std::string>(model_url, '#').front();163 f = string_split<std::string>(f, '?').front();164 model = fs_get_cache_file(string_split<std::string>(f, '/').back());165 }166 } else if (model.empty()) {167 model = model_default;168 }169}170 171const std::vector<ggml_type> kv_cache_types = {172 GGML_TYPE_F32,173 GGML_TYPE_F16,174 GGML_TYPE_BF16,175 GGML_TYPE_Q8_0,176 GGML_TYPE_Q4_0,177 GGML_TYPE_Q4_1,178 GGML_TYPE_IQ4_NL,179 GGML_TYPE_Q5_0,180 GGML_TYPE_Q5_1,181};182 183static ggml_type kv_cache_type_from_str(const std::string & s) {184 for (const auto & type : kv_cache_types) {185 if (ggml_type_name(type) == s) {186 return type;187 }188 }189 throw std::runtime_error("Unsupported cache type: " + s);190}191 192static std::string get_all_kv_cache_types() {193 std::ostringstream msg;194 for (const auto & type : kv_cache_types) {195 msg << ggml_type_name(type) << (&type == &kv_cache_types.back() ? "" : ", ");196 }197 return msg.str();198}199 200//201// CLI argument parsing functions202//203 204static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) {205 std::string arg;206 const std::string arg_prefix = "--";207 common_params & params = ctx_arg.params;208 209 std::unordered_map<std::string, common_arg *> arg_to_options;210 for (auto & opt : ctx_arg.options) {211 for (const auto & arg : opt.args) {212 arg_to_options[arg] = &opt;213 }214 }215 216 // handle environment variables217 for (auto & opt : ctx_arg.options) {218 std::string value;219 if (opt.get_value_from_env(value)) {220 try {221 if (opt.handler_void && (value == "1" || value == "true")) {222 opt.handler_void(params);223 }224 if (opt.handler_int) {225 opt.handler_int(params, std::stoi(value));226 }227 if (opt.handler_string) {228 opt.handler_string(params, value);229 continue;230 }231 } catch (std::exception & e) {232 throw std::invalid_argument(string_format(233 "error while handling environment variable \"%s\": %s\n\n", opt.env, e.what()));234 }235 }236 }237 238 // handle command line arguments239 auto check_arg = [&](int i) {240 if (i+1 >= argc) {241 throw std::invalid_argument("expected value for argument");242 }243 };244 245 for (int i = 1; i < argc; i++) {246 const std::string arg_prefix = "--";247 248 std::string arg = argv[i];249 if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {250 std::replace(arg.begin(), arg.end(), '_', '-');251 }252 if (arg_to_options.find(arg) == arg_to_options.end()) {253 throw std::invalid_argument(string_format("error: invalid argument: %s", arg.c_str()));254 }255 auto opt = *arg_to_options[arg];256 if (opt.has_value_from_env()) {257 fprintf(stderr, "warn: %s environment variable is set, but will be overwritten by command line argument %s\n", opt.env, arg.c_str());258 }259 try {260 if (opt.handler_void) {261 opt.handler_void(params);262 continue;263 }264 265 // arg with single value266 check_arg(i);267 std::string val = argv[++i];268 if (opt.handler_int) {269 opt.handler_int(params, std::stoi(val));270 continue;271 }272 if (opt.handler_string) {273 opt.handler_string(params, val);274 continue;275 }276 277 // arg with 2 values278 check_arg(i);279 std::string val2 = argv[++i];280 if (opt.handler_str_str) {281 opt.handler_str_str(params, val, val2);282 continue;283 }284 } catch (std::exception & e) {285 throw std::invalid_argument(string_format(286 "error while handling argument \"%s\": %s\n\n"287 "usage:\n%s\n\nto show complete usage, run with -h",288 arg.c_str(), e.what(), arg_to_options[arg]->to_string().c_str()));289 }290 }291 292 postprocess_cpu_params(params.cpuparams, nullptr);293 postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams);294 295 postprocess_cpu_params(params.speculative.cpuparams, ¶ms.cpuparams);296 postprocess_cpu_params(params.speculative.cpuparams_batch, ¶ms.cpuparams_batch);297 298 if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {299 throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");300 }301 302 // TODO: refactor model params in a common struct303 common_params_handle_model_default(params.model, params.model_url, params.hf_repo, params.hf_file, params.hf_token, DEFAULT_MODEL_PATH);304 common_params_handle_model_default(params.speculative.model, params.speculative.model_url, params.speculative.hf_repo, params.speculative.hf_file, params.hf_token, "");305 common_params_handle_model_default(params.vocoder.model, params.vocoder.model_url, params.vocoder.hf_repo, params.vocoder.hf_file, params.hf_token, "");306 307 if (params.escape) {308 string_process_escapes(params.prompt);309 string_process_escapes(params.input_prefix);310 string_process_escapes(params.input_suffix);311 for (auto & antiprompt : params.antiprompt) {312 string_process_escapes(antiprompt);313 }314 for (auto & seq_breaker : params.sampling.dry_sequence_breakers) {315 string_process_escapes(seq_breaker);316 }317 }318 319 if (!params.kv_overrides.empty()) {320 params.kv_overrides.emplace_back();321 params.kv_overrides.back().key[0] = 0;322 }323 324 if (params.reranking && params.embedding) {325 throw std::invalid_argument("error: either --embedding or --reranking can be specified, but not both");326 }327 328 if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) {329 throw std::runtime_error(string_format(330 "error: the supplied chat template is not supported: %s%s\n",331 params.chat_template.c_str(),332 params.use_jinja ? "" : "\nnote: llama.cpp was started without --jinja, we only support commonly used templates"333 ));334 }335 336 return true;337}338 339static void common_params_print_usage(common_params_context & ctx_arg) {340 auto print_options = [](std::vector<common_arg *> & options) {341 for (common_arg * opt : options) {342 printf("%s", opt->to_string().c_str());343 }344 };345 346 std::vector<common_arg *> common_options;347 std::vector<common_arg *> sparam_options;348 std::vector<common_arg *> specific_options;349 for (auto & opt : ctx_arg.options) {350 // in case multiple LLAMA_EXAMPLE_* are set, we prioritize the LLAMA_EXAMPLE_* matching current example351 if (opt.is_sparam) {352 sparam_options.push_back(&opt);353 } else if (opt.in_example(ctx_arg.ex)) {354 specific_options.push_back(&opt);355 } else {356 common_options.push_back(&opt);357 }358 }359 printf("----- common params -----\n\n");360 print_options(common_options);361 printf("\n\n----- sampling params -----\n\n");362 print_options(sparam_options);363 // TODO: maybe convert enum llama_example to string364 printf("\n\n----- example-specific params -----\n\n");365 print_options(specific_options);366}367 368static std::vector<ggml_backend_dev_t> parse_device_list(const std::string & value) {369 std::vector<ggml_backend_dev_t> devices;370 auto dev_names = string_split<std::string>(value, ',');371 if (dev_names.empty()) {372 throw std::invalid_argument("no devices specified");373 }374 if (dev_names.size() == 1 && dev_names[0] == "none") {375 devices.push_back(nullptr);376 } else {377 for (const auto & device : dev_names) {378 auto * dev = ggml_backend_dev_by_name(device.c_str());379 if (!dev || ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {380 throw std::invalid_argument(string_format("invalid device: %s", device.c_str()));381 }382 devices.push_back(dev);383 }384 devices.push_back(nullptr);385 }386 return devices;387}388 389static void add_rpc_devices(std::string servers) {390 auto rpc_servers = string_split<std::string>(servers, ',');391 if (rpc_servers.empty()) {392 throw std::invalid_argument("no RPC servers specified");393 }394 ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC");395 if (!rpc_reg) {396 throw std::invalid_argument("failed to find RPC backend");397 }398 typedef ggml_backend_dev_t (*ggml_backend_rpc_add_device_t)(const char * endpoint);399 ggml_backend_rpc_add_device_t ggml_backend_rpc_add_device_fn = (ggml_backend_rpc_add_device_t) ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_device");400 if (!ggml_backend_rpc_add_device_fn) {401 throw std::invalid_argument("failed to find RPC device add function");402 }403 for (const auto & server : rpc_servers) {404 ggml_backend_dev_t dev = ggml_backend_rpc_add_device_fn(server.c_str());405 if (dev) {406 ggml_backend_device_register(dev);407 } else {408 throw std::invalid_argument("failed to register RPC device");409 }410 }411}412 413bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {414 auto ctx_arg = common_params_parser_init(params, ex, print_usage);415 const common_params params_org = ctx_arg.params; // the example can modify the default params416 417 try {418 if (!common_params_parse_ex(argc, argv, ctx_arg)) {419 ctx_arg.params = params_org;420 return false;421 }422 if (ctx_arg.params.usage) {423 common_params_print_usage(ctx_arg);424 if (ctx_arg.print_usage) {425 ctx_arg.print_usage(argc, argv);426 }427 exit(0);428 }429 } catch (const std::invalid_argument & ex) {430 fprintf(stderr, "%s\n", ex.what());431 ctx_arg.params = params_org;432 return false;433 }434 435 return true;436}437 438static std::string list_builtin_chat_templates() {439 std::vector<const char *> supported_tmpl;440 int32_t res = llama_chat_builtin_templates(nullptr, 0);441 supported_tmpl.resize(res);442 res = llama_chat_builtin_templates(supported_tmpl.data(), supported_tmpl.size());443 std::ostringstream msg;444 for (auto & tmpl : supported_tmpl) {445 msg << tmpl << (&tmpl == &supported_tmpl.back() ? "" : ", ");446 }447 return msg.str();448}449 450common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **)) {451 // load dynamic backends452 ggml_backend_load_all();453 454 common_params_context ctx_arg(params);455 ctx_arg.print_usage = print_usage;456 ctx_arg.ex = ex;457 458 std::string sampler_type_chars;459 std::string sampler_type_names;460 for (const auto & sampler : params.sampling.samplers) {461 sampler_type_chars += common_sampler_type_to_chr(sampler);462 sampler_type_names += common_sampler_type_to_str(sampler) + ";";463 }464 sampler_type_names.pop_back();465 466 467 /**468 * filter options by example469 * rules:470 * - all examples inherit options from LLAMA_EXAMPLE_COMMON471 * - if LLAMA_EXAMPLE_* is set (other than COMMON), we only show the option in the corresponding example472 * - if both {LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_*,} are set, we will prioritize the LLAMA_EXAMPLE_* matching current example473 */474 auto add_opt = [&](common_arg arg) {475 if ((arg.in_example(ex) || arg.in_example(LLAMA_EXAMPLE_COMMON)) && !arg.is_exclude(ex)) {476 ctx_arg.options.push_back(std::move(arg));477 }478 };479 480 481 add_opt(common_arg(482 {"-h", "--help", "--usage"},483 "print usage and exit",484 [](common_params & params) {485 params.usage = true;486 }487 ));488 add_opt(common_arg(489 {"--version"},490 "show version and build info",491 [](common_params &) {492 fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);493 fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);494 exit(0);495 }496 ));497 add_opt(common_arg(498 {"--verbose-prompt"},499 string_format("print a verbose prompt before generation (default: %s)", params.verbose_prompt ? "true" : "false"),500 [](common_params & params) {501 params.verbose_prompt = true;502 }503 ));504 add_opt(common_arg(505 {"--no-display-prompt"},506 string_format("don't print prompt at generation (default: %s)", !params.display_prompt ? "true" : "false"),507 [](common_params & params) {508 params.display_prompt = false;509 }510 ).set_examples({LLAMA_EXAMPLE_MAIN}));511 add_opt(common_arg(512 {"-co", "--color"},513 string_format("colorise output to distinguish prompt and user input from generations (default: %s)", params.use_color ? "true" : "false"),514 [](common_params & params) {515 params.use_color = true;516 }517 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_INFILL, LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP}));518 add_opt(common_arg(519 {"-t", "--threads"}, "N",520 string_format("number of threads to use during generation (default: %d)", params.cpuparams.n_threads),521 [](common_params & params, int value) {522 params.cpuparams.n_threads = value;523 if (params.cpuparams.n_threads <= 0) {524 params.cpuparams.n_threads = std::thread::hardware_concurrency();525 }526 }527 ).set_env("LLAMA_ARG_THREADS"));528 add_opt(common_arg(529 {"-tb", "--threads-batch"}, "N",530 "number of threads to use during batch and prompt processing (default: same as --threads)",531 [](common_params & params, int value) {532 params.cpuparams_batch.n_threads = value;533 if (params.cpuparams_batch.n_threads <= 0) {534 params.cpuparams_batch.n_threads = std::thread::hardware_concurrency();535 }536 }537 ));538 add_opt(common_arg(539 {"-C", "--cpu-mask"}, "M",540 "CPU affinity mask: arbitrarily long hex. Complements cpu-range (default: \"\")",541 [](common_params & params, const std::string & mask) {542 params.cpuparams.mask_valid = true;543 if (!parse_cpu_mask(mask, params.cpuparams.cpumask)) {544 throw std::invalid_argument("invalid cpumask");545 }546 }547 ));548 add_opt(common_arg(549 {"-Cr", "--cpu-range"}, "lo-hi",550 "range of CPUs for affinity. Complements --cpu-mask",551 [](common_params & params, const std::string & range) {552 params.cpuparams.mask_valid = true;553 if (!parse_cpu_range(range, params.cpuparams.cpumask)) {554 throw std::invalid_argument("invalid range");555 }556 }557 ));558 add_opt(common_arg(559 {"--cpu-strict"}, "<0|1>",560 string_format("use strict CPU placement (default: %u)\n", (unsigned) params.cpuparams.strict_cpu),561 [](common_params & params, const std::string & value) {562 params.cpuparams.strict_cpu = std::stoul(value);563 }564 ));565 add_opt(common_arg(566 {"--prio"}, "N",567 string_format("set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams.priority),568 [](common_params & params, int prio) {569 if (prio < 0 || prio > 3) {570 throw std::invalid_argument("invalid value");571 }572 params.cpuparams.priority = (enum ggml_sched_priority) prio;573 }574 ));575 add_opt(common_arg(576 {"--poll"}, "<0...100>",577 string_format("use polling level to wait for work (0 - no polling, default: %u)\n", (unsigned) params.cpuparams.poll),578 [](common_params & params, const std::string & value) {579 params.cpuparams.poll = std::stoul(value);580 }581 ));582 add_opt(common_arg(583 {"-Cb", "--cpu-mask-batch"}, "M",584 "CPU affinity mask: arbitrarily long hex. Complements cpu-range-batch (default: same as --cpu-mask)",585 [](common_params & params, const std::string & mask) {586 params.cpuparams_batch.mask_valid = true;587 if (!parse_cpu_mask(mask, params.cpuparams_batch.cpumask)) {588 throw std::invalid_argument("invalid cpumask");589 }590 }591 ));592 add_opt(common_arg(593 {"-Crb", "--cpu-range-batch"}, "lo-hi",594 "ranges of CPUs for affinity. Complements --cpu-mask-batch",595 [](common_params & params, const std::string & range) {596 params.cpuparams_batch.mask_valid = true;597 if (!parse_cpu_range(range, params.cpuparams_batch.cpumask)) {598 throw std::invalid_argument("invalid range");599 }600 }601 ));602 add_opt(common_arg(603 {"--cpu-strict-batch"}, "<0|1>",604 "use strict CPU placement (default: same as --cpu-strict)",605 [](common_params & params, int value) {606 params.cpuparams_batch.strict_cpu = value;607 }608 ));609 add_opt(common_arg(610 {"--prio-batch"}, "N",611 string_format("set process/thread priority : 0-normal, 1-medium, 2-high, 3-realtime (default: %d)\n", params.cpuparams_batch.priority),612 [](common_params & params, int prio) {613 if (prio < 0 || prio > 3) {614 throw std::invalid_argument("invalid value");615 }616 params.cpuparams_batch.priority = (enum ggml_sched_priority) prio;617 }618 ));619 add_opt(common_arg(620 {"--poll-batch"}, "<0|1>",621 "use polling to wait for work (default: same as --poll)",622 [](common_params & params, int value) {623 params.cpuparams_batch.poll = value;624 }625 ));626 add_opt(common_arg(627 {"-lcs", "--lookup-cache-static"}, "FNAME",628 "path to static lookup cache to use for lookup decoding (not updated by generation)",629 [](common_params & params, const std::string & value) {630 params.lookup_cache_static = value;631 }632 ).set_examples({LLAMA_EXAMPLE_LOOKUP}));633 add_opt(common_arg(634 {"-lcd", "--lookup-cache-dynamic"}, "FNAME",635 "path to dynamic lookup cache to use for lookup decoding (updated by generation)",636 [](common_params & params, const std::string & value) {637 params.lookup_cache_dynamic = value;638 }639 ).set_examples({LLAMA_EXAMPLE_LOOKUP}));640 add_opt(common_arg(641 {"-c", "--ctx-size"}, "N",642 string_format("size of the prompt context (default: %d, 0 = loaded from model)", params.n_ctx),643 [](common_params & params, int value) {644 params.n_ctx = value;645 }646 ).set_env("LLAMA_ARG_CTX_SIZE"));647 add_opt(common_arg(648 {"-n", "--predict", "--n-predict"}, "N",649 string_format("number of tokens to predict (default: %d, -1 = infinity, -2 = until context filled)", params.n_predict),650 [](common_params & params, int value) {651 params.n_predict = value;652 }653 ).set_env("LLAMA_ARG_N_PREDICT"));654 add_opt(common_arg(655 {"-b", "--batch-size"}, "N",656 string_format("logical maximum batch size (default: %d)", params.n_batch),657 [](common_params & params, int value) {658 params.n_batch = value;659 }660 ).set_env("LLAMA_ARG_BATCH"));661 add_opt(common_arg(662 {"-ub", "--ubatch-size"}, "N",663 string_format("physical maximum batch size (default: %d)", params.n_ubatch),664 [](common_params & params, int value) {665 params.n_ubatch = value;666 }667 ).set_env("LLAMA_ARG_UBATCH"));668 add_opt(common_arg(669 {"--keep"}, "N",670 string_format("number of tokens to keep from the initial prompt (default: %d, -1 = all)", params.n_keep),671 [](common_params & params, int value) {672 params.n_keep = value;673 }674 ));675 add_opt(common_arg(676 {"--no-context-shift"},677 string_format("disables context shift on inifinite text generation (default: %s)", params.ctx_shift ? "disabled" : "enabled"),678 [](common_params & params) {679 params.ctx_shift = false;680 }681 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY}).set_env("LLAMA_ARG_NO_CONTEXT_SHIFT"));682 add_opt(common_arg(683 {"--chunks"}, "N",684 string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),685 [](common_params & params, int value) {686 params.n_chunks = value;687 }688 ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_RETRIEVAL}));689 add_opt(common_arg(690 {"-fa", "--flash-attn"},691 string_format("enable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled"),692 [](common_params & params) {693 params.flash_attn = true;694 }695 ).set_env("LLAMA_ARG_FLASH_ATTN"));696 add_opt(common_arg(697 {"-p", "--prompt"}, "PROMPT",698 ex == LLAMA_EXAMPLE_MAIN699 ? "prompt to start generation with\nif -cnv is set, this will be used as system prompt"700 : "prompt to start generation with",701 [](common_params & params, const std::string & value) {702 params.prompt = value;703 }704 ).set_excludes({LLAMA_EXAMPLE_SERVER}));705 add_opt(common_arg(706 {"--no-perf"},707 string_format("disable internal libllama performance timings (default: %s)", params.no_perf ? "true" : "false"),708 [](common_params & params) {709 params.no_perf = true;710 params.sampling.no_perf = true;711 }712 ).set_env("LLAMA_ARG_NO_PERF"));713 add_opt(common_arg(714 {"-f", "--file"}, "FNAME",715 "a file containing the prompt (default: none)",716 [](common_params & params, const std::string & value) {717 std::ifstream file(value);718 if (!file) {719 throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));720 }721 // store the external file name in params722 params.prompt_file = value;723 std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));724 if (!params.prompt.empty() && params.prompt.back() == '\n') {725 params.prompt.pop_back();726 }727 }728 ).set_excludes({LLAMA_EXAMPLE_SERVER}));729 add_opt(common_arg(730 {"--in-file"}, "FNAME",731 "an input file (repeat to specify multiple files)",732 [](common_params & params, const std::string & value) {733 std::ifstream file(value);734 if (!file) {735 throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));736 }737 params.in_files.push_back(value);738 }739 ).set_examples({LLAMA_EXAMPLE_IMATRIX}));740 add_opt(common_arg(741 {"-bf", "--binary-file"}, "FNAME",742 "binary file containing the prompt (default: none)",743 [](common_params & params, const std::string & value) {744 std::ifstream file(value, std::ios::binary);745 if (!file) {746 throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));747 }748 // store the external file name in params749 params.prompt_file = value;750 std::ostringstream ss;751 ss << file.rdbuf();752 params.prompt = ss.str();753 fprintf(stderr, "Read %zu bytes from binary file %s\n", params.prompt.size(), value.c_str());754 }755 ).set_excludes({LLAMA_EXAMPLE_SERVER}));756 add_opt(common_arg(757 {"-e", "--escape"},758 string_format("process escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\) (default: %s)", params.escape ? "true" : "false"),759 [](common_params & params) {760 params.escape = true;761 }762 ));763 add_opt(common_arg(764 {"--no-escape"},765 "do not process escape sequences",766 [](common_params & params) {767 params.escape = false;768 }769 ));770 add_opt(common_arg(771 {"-ptc", "--print-token-count"}, "N",772 string_format("print token count every N tokens (default: %d)", params.n_print),773 [](common_params & params, int value) {774 params.n_print = value;775 }776 ).set_examples({LLAMA_EXAMPLE_MAIN}));777 add_opt(common_arg(778 {"--prompt-cache"}, "FNAME",779 "file to cache prompt state for faster startup (default: none)",780 [](common_params & params, const std::string & value) {781 params.path_prompt_cache = value;782 }783 ).set_examples({LLAMA_EXAMPLE_MAIN}));784 add_opt(common_arg(785 {"--prompt-cache-all"},786 "if specified, saves user input and generations to cache as well\n",787 [](common_params & params) {788 params.prompt_cache_all = true;789 }790 ).set_examples({LLAMA_EXAMPLE_MAIN}));791 add_opt(common_arg(792 {"--prompt-cache-ro"},793 "if specified, uses the prompt cache but does not update it",794 [](common_params & params) {795 params.prompt_cache_ro = true;796 }797 ).set_examples({LLAMA_EXAMPLE_MAIN}));798 add_opt(common_arg(799 {"-r", "--reverse-prompt"}, "PROMPT",800 "halt generation at PROMPT, return control in interactive mode\n",801 [](common_params & params, const std::string & value) {802 params.antiprompt.emplace_back(value);803 }804 ).set_examples({LLAMA_EXAMPLE_MAIN}));805 add_opt(common_arg(806 {"-sp", "--special"},807 string_format("special tokens output enabled (default: %s)", params.special ? "true" : "false"),808 [](common_params & params) {809 params.special = true;810 }811 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}));812 add_opt(common_arg(813 {"-cnv", "--conversation"},814 "run in conversation mode:\n"815 "- does not print special tokens and suffix/prefix\n"816 "- interactive mode is also enabled\n"817 "(default: auto enabled if chat template is available)",818 [](common_params & params) {819 params.conversation_mode = COMMON_CONVERSATION_MODE_ENABLED;820 }821 ).set_examples({LLAMA_EXAMPLE_MAIN}));822 add_opt(common_arg(823 {"-no-cnv", "--no-conversation"},824 "force disable conversation mode (default: false)",825 [](common_params & params) {826 params.conversation_mode = COMMON_CONVERSATION_MODE_DISABLED;827 }828 ).set_examples({LLAMA_EXAMPLE_MAIN}));829 add_opt(common_arg(830 {"-i", "--interactive"},831 string_format("run in interactive mode (default: %s)", params.interactive ? "true" : "false"),832 [](common_params & params) {833 params.interactive = true;834 }835 ).set_examples({LLAMA_EXAMPLE_MAIN}));836 add_opt(common_arg(837 {"-if", "--interactive-first"},838 string_format("run in interactive mode and wait for input right away (default: %s)", params.interactive_first ? "true" : "false"),839 [](common_params & params) {840 params.interactive_first = true;841 }842 ).set_examples({LLAMA_EXAMPLE_MAIN}));843 add_opt(common_arg(844 {"-mli", "--multiline-input"},845 "allows you to write or paste multiple lines without ending each in '\\'",846 [](common_params & params) {847 params.multiline_input = true;848 }849 ).set_examples({LLAMA_EXAMPLE_MAIN}));850 add_opt(common_arg(851 {"--in-prefix-bos"},852 "prefix BOS to user inputs, preceding the `--in-prefix` string",853 [](common_params & params) {854 params.input_prefix_bos = true;855 params.enable_chat_template = false;856 }857 ).set_examples({LLAMA_EXAMPLE_MAIN}));858 add_opt(common_arg(859 {"--in-prefix"}, "STRING",860 "string to prefix user inputs with (default: empty)",861 [](common_params & params, const std::string & value) {862 params.input_prefix = value;863 params.enable_chat_template = false;864 }865 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_INFILL}));866 add_opt(common_arg(867 {"--in-suffix"}, "STRING",868 "string to suffix after user inputs with (default: empty)",869 [](common_params & params, const std::string & value) {870 params.input_suffix = value;871 params.enable_chat_template = false;872 }873 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_INFILL}));874 add_opt(common_arg(875 {"--no-warmup"},876 "skip warming up the model with an empty run",877 [](common_params & params) {878 params.warmup = false;879 }880 ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING}));881 add_opt(common_arg(882 {"--spm-infill"},883 string_format(884 "use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. (default: %s)",885 params.spm_infill ? "enabled" : "disabled"886 ),887 [](common_params & params) {888 params.spm_infill = true;889 }890 ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_INFILL}));891 add_opt(common_arg(892 {"--samplers"}, "SAMPLERS",893 string_format("samplers that will be used for generation in the order, separated by \';\'\n(default: %s)", sampler_type_names.c_str()),894 [](common_params & params, const std::string & value) {895 const auto sampler_names = string_split<std::string>(value, ';');896 params.sampling.samplers = common_sampler_types_from_names(sampler_names, true);897 }898 ).set_sparam());899 add_opt(common_arg(900 {"-s", "--seed"}, "SEED",901 string_format("RNG seed (default: %d, use random seed for %d)", params.sampling.seed, LLAMA_DEFAULT_SEED),902 [](common_params & params, const std::string & value) {903 params.sampling.seed = std::stoul(value);904 }905 ).set_sparam());906 add_opt(common_arg(907 {"--sampling-seq", "--sampler-seq"}, "SEQUENCE",908 string_format("simplified sequence for samplers that will be used (default: %s)", sampler_type_chars.c_str()),909 [](common_params & params, const std::string & value) {910 params.sampling.samplers = common_sampler_types_from_chars(value);911 }912 ).set_sparam());913 add_opt(common_arg(914 {"--ignore-eos"},915 "ignore end of stream token and continue generating (implies --logit-bias EOS-inf)",916 [](common_params & params) {917 params.sampling.ignore_eos = true;918 }919 ).set_sparam());920 add_opt(common_arg(921 {"--temp"}, "N",922 string_format("temperature (default: %.1f)", (double)params.sampling.temp),923 [](common_params & params, const std::string & value) {924 params.sampling.temp = std::stof(value);925 params.sampling.temp = std::max(params.sampling.temp, 0.0f);926 }927 ).set_sparam());928 add_opt(common_arg(929 {"--top-k"}, "N",930 string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k),931 [](common_params & params, int value) {932 params.sampling.top_k = value;933 }934 ).set_sparam());935 add_opt(common_arg(936 {"--top-p"}, "N",937 string_format("top-p sampling (default: %.1f, 1.0 = disabled)", (double)params.sampling.top_p),938 [](common_params & params, const std::string & value) {939 params.sampling.top_p = std::stof(value);940 }941 ).set_sparam());942 add_opt(common_arg(943 {"--min-p"}, "N",944 string_format("min-p sampling (default: %.1f, 0.0 = disabled)", (double)params.sampling.min_p),945 [](common_params & params, const std::string & value) {946 params.sampling.min_p = std::stof(value);947 }948 ).set_sparam());949 add_opt(common_arg(950 {"--xtc-probability"}, "N",951 string_format("xtc probability (default: %.1f, 0.0 = disabled)", (double)params.sampling.xtc_probability),952 [](common_params & params, const std::string & value) {953 params.sampling.xtc_probability = std::stof(value);954 }955 ).set_sparam());956 add_opt(common_arg(957 {"--xtc-threshold"}, "N",958 string_format("xtc threshold (default: %.1f, 1.0 = disabled)", (double)params.sampling.xtc_threshold),959 [](common_params & params, const std::string & value) {960 params.sampling.xtc_threshold = std::stof(value);961 }962 ).set_sparam());963 add_opt(common_arg(964 {"--typical"}, "N",965 string_format("locally typical sampling, parameter p (default: %.1f, 1.0 = disabled)", (double)params.sampling.typ_p),966 [](common_params & params, const std::string & value) {967 params.sampling.typ_p = std::stof(value);968 }969 ).set_sparam());970 add_opt(common_arg(971 {"--repeat-last-n"}, "N",972 string_format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sampling.penalty_last_n),973 [](common_params & params, int value) {974 if (value < -1) {975 throw std::runtime_error(string_format("error: invalid repeat-last-n = %d\n", value));976 }977 params.sampling.penalty_last_n = value;978 params.sampling.n_prev = std::max(params.sampling.n_prev, params.sampling.penalty_last_n);979 }980 ).set_sparam());981 add_opt(common_arg(982 {"--repeat-penalty"}, "N",983 string_format("penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)", (double)params.sampling.penalty_repeat),984 [](common_params & params, const std::string & value) {985 params.sampling.penalty_repeat = std::stof(value);986 }987 ).set_sparam());988 add_opt(common_arg(989 {"--presence-penalty"}, "N",990 string_format("repeat alpha presence penalty (default: %.1f, 0.0 = disabled)", (double)params.sampling.penalty_present),991 [](common_params & params, const std::string & value) {992 params.sampling.penalty_present = std::stof(value);993 }994 ).set_sparam());995 add_opt(common_arg(996 {"--frequency-penalty"}, "N",997 string_format("repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)", (double)params.sampling.penalty_freq),998 [](common_params & params, const std::string & value) {999 params.sampling.penalty_freq = std::stof(value);1000 }1001 ).set_sparam());1002 add_opt(common_arg(1003 {"--dry-multiplier"}, "N",1004 string_format("set DRY sampling multiplier (default: %.1f, 0.0 = disabled)", (double)params.sampling.dry_multiplier),1005 [](common_params & params, const std::string & value) {1006 params.sampling.dry_multiplier = std::stof(value);1007 }1008 ).set_sparam());1009 add_opt(common_arg(1010 {"--dry-base"}, "N",1011 string_format("set DRY sampling base value (default: %.2f)", (double)params.sampling.dry_base),1012 [](common_params & params, const std::string & value) {1013 float potential_base = std::stof(value);1014 if (potential_base >= 1.0f)1015 {1016 params.sampling.dry_base = potential_base;1017 }1018 }1019 ).set_sparam());1020 add_opt(common_arg(1021 {"--dry-allowed-length"}, "N",1022 string_format("set allowed length for DRY sampling (default: %d)", params.sampling.dry_allowed_length),1023 [](common_params & params, int value) {1024 params.sampling.dry_allowed_length = value;1025 }1026 ).set_sparam());1027 add_opt(common_arg(1028 {"--dry-penalty-last-n"}, "N",1029 string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable, -1 = context size)", params.sampling.dry_penalty_last_n),1030 [](common_params & params, int value) {1031 if (value < -1) {1032 throw std::runtime_error(string_format("error: invalid dry-penalty-last-n = %d\n", value));1033 }1034 params.sampling.dry_penalty_last_n = value;1035 }1036 ).set_sparam());1037 add_opt(common_arg(1038 {"--dry-sequence-breaker"}, "STRING",1039 string_format("add sequence breaker for DRY sampling, clearing out default breakers (%s) in the process; use \"none\" to not use any sequence breakers\n",1040 params.sampling.dry_sequence_breakers.empty() ? "none" :1041 std::accumulate(std::next(params.sampling.dry_sequence_breakers.begin()),1042 params.sampling.dry_sequence_breakers.end(),1043 std::string("'") + (params.sampling.dry_sequence_breakers[0] == "\n" ? "\\n" : params.sampling.dry_sequence_breakers[0]) + "'",1044 [](const std::string& a, const std::string& b) {1045 std::string formatted_b = (b == "\n") ? "\\n" : b;1046 return a + ", '" + formatted_b + "'";1047 }).c_str()),1048 [](common_params & params, const std::string & value) {1049 static bool defaults_cleared = false;1050 1051 if (!defaults_cleared) {1052 params.sampling.dry_sequence_breakers.clear();1053 defaults_cleared = true;1054 }1055 1056 if (value == "none") {1057 params.sampling.dry_sequence_breakers.clear();1058 } else {1059 params.sampling.dry_sequence_breakers.emplace_back(value);1060 }1061 }1062 ).set_sparam());1063 add_opt(common_arg(1064 {"--dynatemp-range"}, "N",1065 string_format("dynamic temperature range (default: %.1f, 0.0 = disabled)", (double)params.sampling.dynatemp_range),1066 [](common_params & params, const std::string & value) {1067 params.sampling.dynatemp_range = std::stof(value);1068 }1069 ).set_sparam());1070 add_opt(common_arg(1071 {"--dynatemp-exp"}, "N",1072 string_format("dynamic temperature exponent (default: %.1f)", (double)params.sampling.dynatemp_exponent),1073 [](common_params & params, const std::string & value) {1074 params.sampling.dynatemp_exponent = std::stof(value);1075 }1076 ).set_sparam());1077 add_opt(common_arg(1078 {"--mirostat"}, "N",1079 string_format("use Mirostat sampling.\nTop K, Nucleus and Locally Typical samplers are ignored if used.\n"1080 "(default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)", params.sampling.mirostat),1081 [](common_params & params, int value) {1082 params.sampling.mirostat = value;1083 }1084 ).set_sparam());1085 add_opt(common_arg(1086 {"--mirostat-lr"}, "N",1087 string_format("Mirostat learning rate, parameter eta (default: %.1f)", (double)params.sampling.mirostat_eta),1088 [](common_params & params, const std::string & value) {1089 params.sampling.mirostat_eta = std::stof(value);1090 }1091 ).set_sparam());1092 add_opt(common_arg(1093 {"--mirostat-ent"}, "N",1094 string_format("Mirostat target entropy, parameter tau (default: %.1f)", (double)params.sampling.mirostat_tau),1095 [](common_params & params, const std::string & value) {1096 params.sampling.mirostat_tau = std::stof(value);1097 }1098 ).set_sparam());1099 add_opt(common_arg(1100 {"-l", "--logit-bias"}, "TOKEN_ID(+/-)BIAS",1101 "modifies the likelihood of token appearing in the completion,\n"1102 "i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',\n"1103 "or `--logit-bias 15043-1` to decrease likelihood of token ' Hello'",1104 [](common_params & params, const std::string & value) {1105 std::stringstream ss(value);1106 llama_token key;1107 char sign;1108 std::string value_str;1109 try {1110 if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) {1111 const float bias = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f);1112 params.sampling.logit_bias.push_back({key, bias});1113 } else {1114 throw std::invalid_argument("invalid input format");1115 }1116 } catch (const std::exception&) {1117 throw std::invalid_argument("invalid input format");1118 }1119 }1120 ).set_sparam());1121 add_opt(common_arg(1122 {"--grammar"}, "GRAMMAR",1123 string_format("BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '%s')", params.sampling.grammar.c_str()),1124 [](common_params & params, const std::string & value) {1125 params.sampling.grammar = value;1126 }1127 ).set_sparam());1128 add_opt(common_arg(1129 {"--grammar-file"}, "FNAME",1130 "file to read grammar from",1131 [](common_params & params, const std::string & value) {1132 std::ifstream file(value);1133 if (!file) {1134 throw std::runtime_error(string_format("error: failed to open file '%s'\n", value.c_str()));1135 }1136 std::copy(1137 std::istreambuf_iterator<char>(file),1138 std::istreambuf_iterator<char>(),1139 std::back_inserter(params.sampling.grammar)1140 );1141 }1142 ).set_sparam());1143 add_opt(common_arg(1144 {"-j", "--json-schema"}, "SCHEMA",1145 "JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",1146 [](common_params & params, const std::string & value) {1147 params.sampling.grammar = json_schema_to_grammar(json::parse(value));1148 }1149 ).set_sparam());1150 add_opt(common_arg(1151 {"--pooling"}, "{none,mean,cls,last,rank}",1152 "pooling type for embeddings, use model default if unspecified",1153 [](common_params & params, const std::string & value) {1154 /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }1155 else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }1156 else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }1157 else if (value == "last") { params.pooling_type = LLAMA_POOLING_TYPE_LAST; }1158 else if (value == "rank") { params.pooling_type = LLAMA_POOLING_TYPE_RANK; }1159 else { throw std::invalid_argument("invalid value"); }1160 }1161 ).set_examples({LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_RETRIEVAL, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_POOLING"));1162 add_opt(common_arg(1163 {"--attention"}, "{causal,non-causal}",1164 "attention type for embeddings, use model default if unspecified",1165 [](common_params & params, const std::string & value) {1166 /**/ if (value == "causal") { params.attention_type = LLAMA_ATTENTION_TYPE_CAUSAL; }1167 else if (value == "non-causal") { params.attention_type = LLAMA_ATTENTION_TYPE_NON_CAUSAL; }1168 else { throw std::invalid_argument("invalid value"); }1169 }1170 ).set_examples({LLAMA_EXAMPLE_EMBEDDING}));1171 add_opt(common_arg(1172 {"--rope-scaling"}, "{none,linear,yarn}",1173 "RoPE frequency scaling method, defaults to linear unless specified by the model",1174 [](common_params & params, const std::string & value) {1175 /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; }1176 else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }1177 else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }1178 else { throw std::invalid_argument("invalid value"); }1179 }1180 ).set_env("LLAMA_ARG_ROPE_SCALING_TYPE"));1181 add_opt(common_arg(1182 {"--rope-scale"}, "N",1183 "RoPE context scaling factor, expands context by a factor of N",1184 [](common_params & params, const std::string & value) {1185 params.rope_freq_scale = 1.0f / std::stof(value);1186 }1187 ).set_env("LLAMA_ARG_ROPE_SCALE"));1188 add_opt(common_arg(1189 {"--rope-freq-base"}, "N",1190 "RoPE base frequency, used by NTK-aware scaling (default: loaded from model)",1191 [](common_params & params, const std::string & value) {1192 params.rope_freq_base = std::stof(value);1193 }1194 ).set_env("LLAMA_ARG_ROPE_FREQ_BASE"));1195 add_opt(common_arg(1196 {"--rope-freq-scale"}, "N",1197 "RoPE frequency scaling factor, expands context by a factor of 1/N",1198 [](common_params & params, const std::string & value) {1199 params.rope_freq_scale = std::stof(value);1200 }