echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0479
1#include "chat.h"2 3#include "chat-auto-parser-helpers.h"4#include "chat-auto-parser.h"5#include "chat-peg-parser.h"6#include "common.h"7#include "ggml.h"8#include "json-schema-to-grammar.h"9#include "log.h"10 11#include "jinja/value.h"12#include "jinja/runtime.h"13#include "jinja/caps.h"14#include "peg-parser.h"15 16#include "nlohmann/json.hpp"17 18#include <cstdio>19#include <cstdlib>20#include <ctime>21#include <exception>22#include <functional>23 24#include <optional>25#include <sstream>26#include <stdexcept>27#include <string>28#include <utility>29#include <vector>30 31using json = nlohmann::ordered_json;32 33static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {34 auto time = std::chrono::system_clock::to_time_t(now);35 auto local_time = *std::localtime(&time);36 std::ostringstream ss;37 ss << std::put_time(&local_time, format.c_str());38 auto res = ss.str();39 return res;40}41 42static json safe_args_parse(const std::string & to_parse) {43 std::string stripped = to_parse;44 if (to_parse.at(0) == '"' && to_parse.at(to_parse.length() - 1) == '"') {45 stripped = to_parse.substr(1, to_parse.length() - 1);46 }47 try {48 return json::parse(stripped);49 } catch (json::exception & e) {50 return stripped;51 }52}53 54static std::string string_diff(const std::string & last, const std::string & current) {55 if (last.empty()) {56 return current;57 }58 if (!string_starts_with(current, last)) {59 if (string_starts_with(last, current)) {60 // This happens if the last generation ended on a partial stop word (not erased),61 // and the current ended on a stop word (erased).62 return "";63 }64 throw std::runtime_error("Invalid diff: '" + last + "' not found at start of '" + current + "'");65 }66 return current.substr(last.size());67}68 69static bool has_content_or_tool_calls(const common_chat_msg & msg) {70 return !msg.content.empty() || !msg.tool_calls.empty();71}72 73json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {74 if (!content.empty() && !content_parts.empty()) {75 throw std::runtime_error("Cannot specify both content and content_parts");76 }77 json jmsg {78 {"role", role},79 };80 if (!content.empty()) {81 jmsg["content"] = content;82 } else if (!content_parts.empty()) {83 if (concat_typed_text) {84 std::string text;85 bool last_was_media_marker = false;86 // join parts with newline, do not add newline before or after media markers87 for (const auto & part : content_parts) {88 bool add_new_line = true;89 if (part.type == "text") {90 add_new_line = !last_was_media_marker && !text.empty();91 last_was_media_marker = false;92 } else if (part.type == "media_marker") {93 add_new_line = false;94 last_was_media_marker = true;95 } else {96 LOG_WRN("Ignoring content part type: %s\n", part.type.c_str());97 continue;98 }99 100 if (add_new_line) {101 text += '\n';102 }103 104 text += part.text;105 }106 jmsg["content"] = text;107 } else {108 auto & parts = jmsg["content"] = json::array();109 for (const auto & part : content_parts) {110 parts.push_back({111 {"type", part.type},112 {"text", part.text},113 });114 }115 }116 } else {117 jmsg["content"] = "";118 }119 if (!reasoning_content.empty()) {120 jmsg["reasoning_content"] = reasoning_content;121 }122 if (!tool_name.empty()) {123 jmsg["name"] = tool_name;124 }125 if (!tool_call_id.empty()) {126 jmsg["tool_call_id"] = tool_call_id;127 }128 if (!tool_calls.empty()) {129 jmsg["tool_calls"] = json::array();130 auto & jtool_calls = jmsg["tool_calls"];131 for (const auto & tool_call : tool_calls) {132 json tc {133 {"type", "function"},134 {"function", {135 {"name", tool_call.name},136 {"arguments", json(tool_call.arguments)},137 }},138 };139 if (!tool_call.id.empty()) {140 tc["id"] = tool_call.id;141 }142 // Some templates generate and require an id (sometimes in a very specific format, e.g. Mistral Nemo).143 // We only generate a random id for the ones that don't generate one by themselves144 // (they also won't get to see it as their template likely doesn't use it, so it's all for the client)145 // {"id", tc.id.empty() ? gen_tool_call_id() : tc.id},146 jtool_calls.push_back(tc);147 }148 }149 150 return jmsg;151}152 153std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const common_chat_msg & msg_prv,154 const common_chat_msg & msg_new) {155 std::vector<common_chat_msg_diff> diffs;156 if (msg_new.tool_calls.size() > msg_prv.tool_calls.size()) {157 diffs.reserve(msg_new.tool_calls.size() - msg_prv.tool_calls.size() + 3);158 } else {159 diffs.reserve(3);160 }161 162 // TODO: these can become expensive for long messages - how to optimize?163 if (msg_prv.reasoning_content != msg_new.reasoning_content) {164 auto & diff = diffs.emplace_back();165 diff.reasoning_content_delta = string_diff(msg_prv.reasoning_content, msg_new.reasoning_content);166 }167 if (msg_prv.content != msg_new.content) {168 auto & diff = diffs.emplace_back();169 diff.content_delta = string_diff(msg_prv.content, msg_new.content);170 }171 172 if (msg_new.tool_calls.size() < msg_prv.tool_calls.size()) {173 std::string err = "Invalid diff: now finding less tool calls!\n";174 err += " Previous (" + std::to_string(msg_prv.tool_calls.size()) + "):\n";175 for (const auto & tc : msg_prv.tool_calls) {176 err += " - name: '" + tc.name + "', args: '" + tc.arguments + "'\n";177 }178 err += " Current (" + std::to_string(msg_new.tool_calls.size()) + "):\n";179 for (const auto & tc : msg_new.tool_calls) {180 err += " - name: '" + tc.name + "', args: '" + tc.arguments + "'\n";181 }182 err += " Current msg text content:\n" + msg_new.content + "\n";183 throw std::runtime_error(err);184 }185 186 if (!msg_prv.tool_calls.empty()) {187 const auto idx = msg_prv.tool_calls.size() - 1;188 const auto & pref = msg_prv.tool_calls[idx];189 const auto & newf = msg_new.tool_calls[idx];190 // Allow tool name to change during incremental parsing:191 // - empty -> non-empty (initial discovery)192 // - prefix -> longer string (name grows as more input is parsed)193 if (pref.name != newf.name && !pref.name.empty() && !newf.name.empty()) {194 // Check if one is a prefix of the other (for incremental parsing where names grow or shrink)195 bool is_prefix = (newf.name.rfind(pref.name, 0) == 0);196 if (!is_prefix) {197 LOG_ERR("Tool call mismatch: prev='%s' new='%s'\n", pref.name.c_str(), newf.name.c_str());198 throw std::runtime_error("Invalid diff: tool call mismatch!");199 }200 }201 const auto args_diff = string_diff(pref.arguments, newf.arguments);202 if (!args_diff.empty() || pref.id != newf.id || pref.name != newf.name) {203 auto & diff = diffs.emplace_back();204 diff.tool_call_index = idx;205 if (pref.id != newf.id || pref.name != newf.name) {206 diff.tool_call_delta.id = newf.id;207 diff.tool_call_delta.name = newf.name;208 }209 diff.tool_call_delta.arguments = args_diff;210 }211 }212 for (size_t idx = msg_prv.tool_calls.size(); idx < msg_new.tool_calls.size(); ++idx) {213 auto & diff = diffs.emplace_back();214 diff.tool_call_index = idx;215 diff.tool_call_delta = msg_new.tool_calls[idx];216 }217 218 return diffs;219}220 221using chat_template_caps = jinja::caps;222 223struct common_chat_templates {224 bool add_bos;225 bool add_eos;226 bool has_explicit_template; // Model had builtin template or template overridden was specified.227 std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)228 std::unique_ptr<common_chat_template> template_tool_use;229};230 231common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {232 if (tool_choice == "auto") {233 return COMMON_CHAT_TOOL_CHOICE_AUTO;234 }235 if (tool_choice == "none") {236 return COMMON_CHAT_TOOL_CHOICE_NONE;237 }238 if (tool_choice == "required") {239 return COMMON_CHAT_TOOL_CHOICE_REQUIRED;240 }241 throw std::invalid_argument("Invalid tool_choice: " + tool_choice);242}243 244bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates) {245 common_chat_templates_inputs inputs;246 inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;247 common_chat_msg msg;248 msg.role = "user";249 msg.content = "test";250 inputs.messages = { msg };251 inputs.enable_thinking = true;252 inputs.add_generation_prompt = true;253 inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;254 255 auto params = common_chat_templates_apply(chat_templates, inputs);256 return params.supports_thinking;257}258 259std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messages) {260 std::vector<common_chat_msg> msgs;261 262 try {263 if (!messages.is_array()) {264 throw std::invalid_argument("Expected 'messages' to be an array, got " + messages.dump());265 }266 267 for (const auto & message : messages) {268 if (!message.is_object()) {269 throw std::invalid_argument("Expected 'message' to be an object, got " + message.dump());270 }271 272 common_chat_msg msg;273 if (!message.contains("role")) {274 throw std::invalid_argument("Missing 'role' in message: " + message.dump());275 }276 msg.role = message.at("role");277 278 auto has_content = message.contains("content");279 auto has_tool_calls = message.contains("tool_calls");280 if (has_content) {281 const auto & content = message.at("content");282 if (content.is_string()) {283 msg.content = content;284 } else if (content.is_array()) {285 for (const auto & part : content) {286 if (!part.contains("type")) {287 throw std::invalid_argument("Missing content part type: " + part.dump());288 }289 const auto & type = part.at("type");290 if (type != "text" && type != "media_marker") {291 throw std::invalid_argument("Unsupported content part type: " + type.dump());292 }293 common_chat_msg_content_part msg_part;294 msg_part.type = type;295 msg_part.text = part.at("text");296 msg.content_parts.push_back(msg_part);297 }298 } else if (!content.is_null()) {299 throw std::invalid_argument("Invalid 'content' type: expected string or array, got " +300 content.dump() +301 " (ref: https://github.com/ggml-org/llama.cpp/issues/8367)");302 }303 }304 if (has_tool_calls) {305 for (const auto & tool_call : message.at("tool_calls")) {306 common_chat_tool_call tc;307 if (!tool_call.contains("type")) {308 throw std::invalid_argument("Missing tool call type: " + tool_call.dump());309 }310 const auto & type = tool_call.at("type");311 if (type != "function") {312 throw std::invalid_argument("Unsupported tool call type: " + tool_call.dump());313 }314 if (!tool_call.contains("function")) {315 throw std::invalid_argument("Missing tool call function: " + tool_call.dump());316 }317 const auto & fc = tool_call.at("function");318 if (!fc.contains("name")) {319 throw std::invalid_argument("Missing tool call name: " + tool_call.dump());320 }321 tc.name = fc.at("name");322 const auto & args = fc.at("arguments");323 if (args.is_string()) {324 tc.arguments = args;325 } else {326 tc.arguments = args.dump();327 }328 if (tool_call.contains("id")) {329 tc.id = tool_call.at("id");330 }331 msg.tool_calls.push_back(tc);332 }333 }334 if (!has_content && !has_tool_calls) {335 throw std::invalid_argument(336 "Expected 'content' or 'tool_calls' (ref: https://github.com/ggml-org/llama.cpp/issues/8367 & "337 "https://github.com/ggml-org/llama.cpp/issues/12279)");338 }339 if (message.contains("reasoning_content")) {340 msg.reasoning_content = message.at("reasoning_content");341 }342 if (message.contains("name")) {343 msg.tool_name = message.at("name");344 }345 if (message.contains("tool_call_id")) {346 msg.tool_call_id = message.at("tool_call_id");347 }348 349 msgs.push_back(msg);350 }351 } catch (const std::exception & e) {352 // @ngxson : disable otherwise it's bloating the API response353 // printf("%s\n", std::string("; messages = ") + messages.dump(2));354 throw std::runtime_error("Failed to parse messages: " + std::string(e.what()));355 }356 357 return msgs;358}359 360static json render_message_to_json(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {361 if (!c.supports_string_content && !c.supports_typed_content) {362 LOG_WRN("%s: Neither string content nor typed content is supported by the template. This is unexpected and may lead to issues.\n", __func__);363 }364 365 bool only_string_accepted = c.supports_string_content && !c.supports_typed_content;366 bool only_typed_accepted = !c.supports_string_content && c.supports_typed_content;367 368 json messages = json::array();369 for (const auto & msg : msgs) {370 if (only_string_accepted) {371 json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ true);372 messages.push_back(jmsg);373 } else if (only_typed_accepted) {374 json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);375 if (jmsg.at("content").is_string()) {376 jmsg["content"] = json::array({377 json{378 {"type", "text"},379 {"text", jmsg.at("content").get<std::string>()},380 }381 });382 }383 messages.push_back(jmsg);384 } else {385 json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);386 messages.push_back(jmsg);387 }388 }389 return messages;390}391 392// DEPRECATED: only used in tests393json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {394 jinja::caps c;395 c.supports_string_content = true;396 c.supports_typed_content = !concat_typed_text;397 return render_message_to_json(msgs, c);398}399 400std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & tools) {401 std::vector<common_chat_tool> result;402 403 try {404 if (!tools.is_null()) {405 if (!tools.is_array()) {406 throw std::invalid_argument("Expected 'tools' to be an array, got " + tools.dump());407 }408 for (const auto & tool : tools) {409 if (!tool.contains("type")) {410 throw std::invalid_argument("Missing tool type: " + tool.dump());411 }412 const auto & type = tool.at("type");413 if (!type.is_string() || type != "function") {414 throw std::invalid_argument("Unsupported tool type: " + tool.dump());415 }416 if (!tool.contains("function")) {417 throw std::invalid_argument("Missing tool function: " + tool.dump());418 }419 420 const auto & function = tool.at("function");421 result.push_back({422 /* .name = */ function.at("name"),423 /* .description = */ function.value("description", ""),424 /* .parameters = */ function.value("parameters", json::object()).dump(),425 });426 }427 }428 } catch (const std::exception & e) {429 throw std::runtime_error("Failed to parse tools: " + std::string(e.what()) + "; tools = " + tools.dump(2));430 }431 432 return result;433}434 435json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools) {436 if (tools.empty()) {437 return json();438 }439 440 auto result = json::array();441 for (const auto & tool : tools) {442 result.push_back({443 { "type", "function" },444 { "function",445 {446 { "name", tool.name },447 { "description", tool.description },448 { "parameters", json::parse(tool.parameters) },449 } },450 });451 }452 return result;453}454 455json common_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {456 json delta = json::object();457 if (!diff.reasoning_content_delta.empty()) {458 delta["reasoning_content"] = diff.reasoning_content_delta;459 }460 if (!diff.content_delta.empty()) {461 delta["content"] = diff.content_delta;462 }463 if (diff.tool_call_index != std::string::npos) {464 json tool_call;465 tool_call["index"] = diff.tool_call_index;466 if (!diff.tool_call_delta.id.empty()) {467 tool_call["id"] = diff.tool_call_delta.id;468 tool_call["type"] = "function";469 }470 if (!diff.tool_call_delta.name.empty() || !diff.tool_call_delta.arguments.empty()) {471 json function = json::object();472 if (!diff.tool_call_delta.name.empty()) {473 function["name"] = diff.tool_call_delta.name;474 }475 if (!diff.tool_call_delta.arguments.empty()) {476 function["arguments"] = diff.tool_call_delta.arguments;477 }478 tool_call["function"] = function;479 }480 delta["tool_calls"] = json::array({ tool_call });481 }482 return delta;483}484 485bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {486 if (use_jinja) {487 try {488 common_chat_msg msg;489 msg.role = "user";490 msg.content = "test";491 492 auto tmpls = common_chat_templates_init(/* model= */ nullptr, tmpl);493 494 common_chat_templates_inputs inputs;495 inputs.messages = { msg };496 497 common_chat_templates_apply(tmpls.get(), inputs);498 return true;499 } catch (const std::exception & e) {500 LOG_ERR("%s: failed to apply template: %s\n", __func__, e.what());501 return false;502 }503 }504 llama_chat_message chat[] = {505 { "user", "test" }506 };507 const int res = llama_chat_apply_template(tmpl.c_str(), chat, 1, true, nullptr, 0);508 return res >= 0;509}510 511std::string common_chat_format_single(const struct common_chat_templates * tmpls,512 const std::vector<common_chat_msg> & past_msg,513 const common_chat_msg & new_msg,514 bool add_ass,515 bool use_jinja) {516 common_chat_templates_inputs inputs;517 inputs.use_jinja = use_jinja;518 inputs.add_bos = tmpls->add_bos;519 inputs.add_eos = tmpls->add_eos;520 521 std::string fmt_past_msg;522 if (!past_msg.empty()) {523 inputs.messages = past_msg;524 inputs.add_generation_prompt = false;525 fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt;526 }527 std::ostringstream ss;528 // if the past_msg ends with a newline, we must preserve it in the formatted version529 if (add_ass && !fmt_past_msg.empty() && fmt_past_msg.back() == '\n') {530 ss << "\n";531 };532 // format chat with new_msg533 inputs.messages.push_back(new_msg);534 inputs.add_generation_prompt = add_ass;535 auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt;536 // get the diff part537 ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());538 return ss.str();539}540 541std::string common_chat_format_example(const struct common_chat_templates * tmpls,542 bool use_jinja,543 const std::map<std::string, std::string> & chat_template_kwargs) {544 common_chat_templates_inputs inputs;545 inputs.use_jinja = use_jinja;546 inputs.add_bos = tmpls->add_bos;547 inputs.add_eos = tmpls->add_eos;548 inputs.chat_template_kwargs = chat_template_kwargs;549 auto add_simple_msg = [&](auto role, auto content) {550 common_chat_msg msg;551 msg.role = role;552 msg.content = content;553 inputs.messages.push_back(msg);554 };555 add_simple_msg("system", "You are a helpful assistant");556 add_simple_msg("user", "Hello");557 add_simple_msg("assistant", "Hi there");558 add_simple_msg("user", "How are you?");559 return common_chat_templates_apply(tmpls, inputs).prompt;560}561 562#define CHATML_TEMPLATE_SRC \563 "{%- for message in messages -%}\n" \564 " {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' -}}\n" \565 "{%- endfor -%}\n" \566 "{%- if add_generation_prompt -%}\n" \567 " {{- '<|im_start|>assistant\n' -}}\n" \568 "{%- endif -%}"569 570void common_chat_templates_free(struct common_chat_templates * tmpls) {571 delete tmpls;572}573 574bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) {575 return tmpls->has_explicit_template;576}577 578std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {579 if (!variant.empty()) {580 if (variant == "tool_use") {581 if (tmpls->template_tool_use) {582 return tmpls->template_tool_use->source();583 }584 return "";585 }586 LOG_DBG("%s: unknown template variant: %s\n", __func__, variant.c_str());587 }588 return tmpls->template_default->source();589}590 591common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,592 const std::string & chat_template_override,593 const std::string & bos_token_override,594 const std::string & eos_token_override) {595 std::string default_template_src;596 std::string template_tool_use_src;597 598 bool has_explicit_template = !chat_template_override.empty();599 if (chat_template_override.empty()) {600 GGML_ASSERT(model != nullptr);601 const auto * str = llama_model_chat_template(model, /* name */ nullptr);602 if (str) {603 default_template_src = str;604 has_explicit_template = true;605 }606 str = llama_model_chat_template(model, /* name */ "tool_use");607 if (str) {608 template_tool_use_src = str;609 has_explicit_template = true;610 }611 } else {612 default_template_src = chat_template_override;613 }614 if (default_template_src.empty() || default_template_src == "chatml") {615 if (!template_tool_use_src.empty()) {616 default_template_src = template_tool_use_src;617 } else {618 default_template_src = CHATML_TEMPLATE_SRC;619 }620 }621 622 // TODO @ngxson : this is a temporary hack to prevent chat template from throwing an error623 // Ref: https://github.com/ggml-org/llama.cpp/pull/15230#issuecomment-3173959633624 if (default_template_src.find("<|channel|>") != std::string::npos625 // search for the error message and patch it626 && default_template_src.find("in message.content or") != std::string::npos) {627 string_replace_all(default_template_src,628 "{%- if \"<|channel|>analysis<|message|>\" in message.content or "629 "\"<|channel|>final<|message|>\" in message.content %}",630 "{%- if false %}");631 }632 633 // TODO @aldehir : this is a temporary fix, pending Minja changes634 // Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664635 if (default_template_src.find("[TOOL_CALLS]") != std::string::npos636 // search for the error message and patch it637 && default_template_src.find("if (message['content'] is none or") != std::string::npos) {638 string_replace_all(default_template_src,639 "{%- if (message['content'] is none or message['content'] == '' or "640 "message['content']|length == 0) and (message['tool_calls'] is not defined or "641 "message['tool_calls'] is none or message['tool_calls']|length == 0) %}",642 "{%- if false %}");643 }644 645 std::string token_bos = bos_token_override;646 std::string token_eos = eos_token_override;647 bool add_bos = false;648 bool add_eos = false;649 if (model) {650 const auto * vocab = llama_model_get_vocab(model);651 const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {652 if (token == LLAMA_TOKEN_NULL) {653 if (default_template_src.find(jinja_variable_name) != std::string::npos ||654 template_tool_use_src.find(jinja_variable_name) != std::string::npos) {655 LOG_WRN(656 "common_chat_templates_init: warning: vocab does not have a %s token, jinja template won't "657 "work as intended.\n",658 name);659 }660 return std::string();661 }662 return common_token_to_piece(vocab, token, true);663 };664 token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");665 token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");666 add_bos = llama_vocab_get_add_bos(vocab);667 add_eos = llama_vocab_get_add_eos(vocab);668 }669 common_chat_templates_ptr tmpls(new common_chat_templates());670 tmpls->has_explicit_template = has_explicit_template;671 tmpls->add_bos = add_bos;672 tmpls->add_eos = add_eos;673 try {674 tmpls->template_default = std::make_unique<common_chat_template>(default_template_src, token_bos, token_eos);675 } catch (const std::exception & e) {676 LOG_ERR("%s: error: %s\n", __func__, e.what());677 LOG_ERR("%s: failed to initialize chat template\n", __func__);678 LOG_ERR("%s: please consider disabling jinja via --no-jinja, or using another chat template\n", __func__);679 throw e;680 }681 if (!template_tool_use_src.empty()) {682 try {683 tmpls->template_tool_use = std::make_unique<common_chat_template>(template_tool_use_src, token_bos, token_eos);684 } catch (const std::exception & e) {685 LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());686 }687 }688 return tmpls;689}690 691const char * common_chat_format_name(common_chat_format format) {692 switch (format) {693 case COMMON_CHAT_FORMAT_CONTENT_ONLY:694 return "Content-only";695 case COMMON_CHAT_FORMAT_PEG_SIMPLE:696 return "peg-simple";697 case COMMON_CHAT_FORMAT_PEG_NATIVE:698 return "peg-native";699 case COMMON_CHAT_FORMAT_PEG_GEMMA4:700 return "peg-gemma4";701 default:702 throw std::runtime_error("Unknown chat format");703 }704}705 706const char * common_reasoning_format_name(common_reasoning_format format) {707 switch (format) {708 case COMMON_REASONING_FORMAT_NONE:709 return "none";710 case COMMON_REASONING_FORMAT_AUTO:711 return "auto";712 case COMMON_REASONING_FORMAT_DEEPSEEK:713 return "deepseek";714 case COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY:715 return "deepseek-legacy";716 default:717 throw std::runtime_error("Unknown reasoning format");718 }719}720 721common_reasoning_format common_reasoning_format_from_name(const std::string & format) {722 if (format == "none") {723 return COMMON_REASONING_FORMAT_NONE;724 }725 if (format == "auto") {726 return COMMON_REASONING_FORMAT_AUTO;727 }728 if (format == "deepseek") {729 return COMMON_REASONING_FORMAT_DEEPSEEK;730 }731 if (format == "deepseek-legacy") {732 return COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY;733 }734 throw std::runtime_error("Unknown reasoning format: " + format);735}736 737static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {738 for (const auto & tool : tools) {739 if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {740 LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());741 continue;742 }743 fn(tool);744 }745}746 747static void foreach_parameter(const json & function,748 const std::function<void(const std::string &, const json &, bool)> & fn) {749 if (!function.contains("parameters") || !function.at("parameters").is_object()) {750 return;751 }752 const auto & params = function.at("parameters");753 if (!params.contains("properties") || !params.at("properties").is_object()) {754 return;755 }756 const auto & props = params.at("properties");757 std::set<std::string> required;758 if (params.contains("required") && params.at("required").is_array()) {759 params.at("required").get_to(required);760 }761 for (const auto & [name, prop] : props.items()) {762 bool is_required = (required.find(name) != required.end());763 fn(name, prop, is_required);764 }765}766 767static std::string common_chat_template_direct_apply_impl(768 const common_chat_template & tmpl,769 const autoparser::generation_params & inputs,770 const std::optional<json> & messages_override = std::nullopt,771 const std::optional<json> & tools_override = std::nullopt,772 const std::optional<json> & additional_context = std::nullopt) {773 jinja::context ctx(tmpl.source());774 775 nlohmann::ordered_json inp = nlohmann::ordered_json{776 {"messages", messages_override.has_value() ? *messages_override : inputs.messages},777 {"bos_token", tmpl.bos_token()},778 {"eos_token", tmpl.eos_token()},779 {"enable_thinking", inputs.enable_thinking},780 };781 if (tools_override.has_value() || !inputs.tools.empty()) {782 inp["tools"] = tools_override.has_value() ? *tools_override : inputs.tools;783 }784 if (inputs.extra_context.is_object()) {785 // TODO: do we need to merge, or replacing is fine?786 for (const auto & [k, v] : inputs.extra_context.items()) {787 inp[k] = v;788 }789 }790 if (additional_context.has_value()) {791 // TODO: merge properly instead of overwriting (matching old behavior)792 for (const auto & [k, v] : additional_context->items()) {793 inp[k] = v;794 }795 }796 if (inputs.add_generation_prompt) {797 inp["add_generation_prompt"] = true;798 }799 800 jinja::global_from_json(ctx, inp, inputs.mark_input);801 802 // render803 jinja::runtime runtime(ctx);804 const jinja::value results = runtime.execute(tmpl.prog);805 auto parts = jinja::runtime::gather_string_parts(results);806 807 std::string result = parts->as_string().str();808 809 // TODO: improve this later810 if (inputs.add_bos && string_starts_with(result, tmpl.bos_token())) {811 result = result.substr(tmpl.bos_token().size());812 }813 if (inputs.add_eos && string_ends_with(result, tmpl.eos_token())) {814 result = result.substr(0, result.size() - tmpl.eos_token().size());815 }816 return result;817}818 819std::string common_chat_template_direct_apply(820 const common_chat_template & tmpl,821 const autoparser::generation_params & inputs) {822 return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);823}824 825static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,826 const autoparser::generation_params & inputs) {827 common_chat_params data;828 829 // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja830 auto adjusted_messages = json::array();831 for (const auto & msg : inputs.messages) {832 auto role = msg.value("role", "");833 if (role != "system" && role != "assistant") {834 // Only adjust system and assistant messages. Interestingly, the system message may contain thinking.835 adjusted_messages.push_back(msg);836 continue;837 }838 839 auto content = json::array();840 841 // If message contains `reasoning_content`, add it as a block of type `thinking`842 if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {843 content.push_back({844 { "type", "thinking" },845 { "thinking", msg.at("reasoning_content").get<std::string>() },846 });847 }848 849 // If message contains `content`, add it as a block of type `text`850 if (msg.contains("content")) {851 if (msg.at("content").is_string()) {852 content.push_back({853 { "type", "text" },854 { "text", msg.at("content").get<std::string>() },855 });856 } else if (msg.at("content").is_array()) {857 auto blocks = msg.at("content");858 content.insert(content.end(), blocks.begin(), blocks.end());859 }860 }861 862 auto adjusted = msg;863 adjusted["content"] = content;864 adjusted.erase("reasoning_content");865 adjusted_messages.push_back(adjusted);866 }867 868 auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();869 auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();870 auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;871 auto include_grammar = true;872 873 data.supports_thinking = true;874 data.thinking_start_tag = "[THINK]";875 data.thinking_end_tag = "[/THINK]";876 data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);877 data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;878 data.preserved_tokens = {879 "[THINK]",880 "[/THINK]",881 "[TOOL_CALLS]",882 "[ARGS]",883 };884 885 auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {886 auto generation_prompt = p.prefix(inputs.generation_prompt, "[THINK]");887 auto reasoning =888 extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();889 890 // Response format parser891 if (has_response_format) {892 // Ministral wants to emit json surrounded by code fences893 return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");894 }895 896 // Tool call parser897 if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {898 auto tool_choice = p.choice();899 foreach_function(inputs.tools, [&](const json & tool) {900 const auto & function = tool.at("function");901 std::string name = function.at("name");902 const auto & schema = function.at("parameters");903 904 tool_choice |=905 p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +906 p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));907 });908 909 auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;910 auto max_calls = inputs.parallel_tool_calls ? -1 : 1;911 auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));912 913 return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);914 }915 916 // Content only parser917 include_grammar = false;918 return generation_prompt + (reasoning << p.content(p.rest()));919 });920 921 data.parser = parser.save();922 923 if (include_grammar) {924 data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;925 926 data.grammar = build_grammar([&](const common_grammar_builder & builder) {927 foreach_function(inputs.tools, [&](const json & tool) {928 const auto & function = tool.at("function");929 auto schema = function.at("parameters");930 builder.resolve_refs(schema);931 });932 if (has_response_format) {933 auto schema = inputs.json_schema;934 builder.resolve_refs(schema);935 }936 parser.build_grammar(builder, data.grammar_lazy);937 });938 939 data.grammar_triggers = {940 { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }941 };942 }943 944 return data;945}946 947static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,948 const autoparser::generation_params & inputs) {949 common_chat_params data;950 951 // Copy reasoning to the "thinking" field as expected by the gpt-oss template952 auto adjusted_messages = json::array();953 for (auto msg : inputs.messages) {954 if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {955 msg["thinking"] = msg.at("reasoning_content");956 if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {957 msg.erase("content");958 }959 }960 adjusted_messages.push_back(msg);961 }962 963 auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);964 965 // Check if we need to replace the return token with end token during966 // inference and without generation prompt. For more details see:967 // https://github.com/ggml-org/llama.cpp/issues/15417968 if (inputs.is_inference && !inputs.add_generation_prompt) {969 static constexpr std::string_view return_token = "<|return|>";970 static constexpr std::string_view end_token = "<|end|>";971 if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) {972 prompt.replace(pos, return_token.length(), end_token);973 }974 }975 976 data.prompt = prompt;977 data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;978 data.supports_thinking = true;979 980 // These special tokens are required to parse properly, so we include them981 // even if parse_tool_calls is false.982 data.preserved_tokens = {983 "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",984 };985 986 auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();987 auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();988 auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);989 auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;990 991 auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {992 auto start = p.rule("start", p.literal("<|start|>assistant"));993 auto end = p.rule("end", p.literal("<|end|>"));994 auto content = p.rule("message-content", p.until("<|end|>"));995 auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));996 auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);997 998 // Occasionally, gpt-oss-20b will prefix channels with this commentary999 auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));1000 auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");1001 1002 if (extract_reasoning) {1003 p.rule("analysis", start_analysis + p.reasoning(content) + end);1004 } else {1005 p.rule("analysis", p.content(start_analysis + content + end));1006 }1007 1008 auto analysis = p.ref("analysis");1009 auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);1010 auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));1011 1012 // Consume any unsolicited tool calls, e.g. builtin functions1013 auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));1014 1015 auto any = p.rule("any", preamble | analysis);1016 1017 if (has_response_format) {1018 auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);1019 auto response_format = p.rule("response-format",1020 p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +1021 p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));1022 1023 return p.zero_or_more(start + analysis) + start + response_format;1024 }1025 1026 if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {1027 auto tool_choice = p.choice();1028 1029 foreach_function(inputs.tools, [&](const json & tool) {1030 const auto & function = tool.at("function");1031 std::string name = function.at("name");1032 const auto & params = function.at("parameters");1033 1034 auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));1035 auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);1036 auto args = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params));1037 1038 // recipient in role header1039 // <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS1040 auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);1041 1042 // recipient in channel header1043 // <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS1044 auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);1045 1046 tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel);1047 });1048 1049 auto tool_call = p.trigger_rule("tool-call", tool_choice);1050 1051 if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {1052 return p.zero_or_more(start + any) + start + tool_call;1053 }1054 1055 return p.zero_or_more(start + any) + start + (tool_call | final_msg);1056 }1057 1058 return p.zero_or_more(start + any) + start + (final_msg | unsolicited);1059 });1060 1061 data.parser = parser.save();1062 1063 if (include_grammar) {1064 data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));1065 data.grammar = build_grammar([&](const common_grammar_builder & builder) {1066 foreach_function(inputs.tools, [&](const json & tool) {1067 const auto & function = tool.at("function");1068 auto schema = function.at("parameters");1069 builder.resolve_refs(schema);1070 });1071 if (has_response_format) {1072 auto schema = inputs.json_schema;1073 builder.resolve_refs(schema);1074 }1075 parser.build_grammar(builder, data.grammar_lazy);1076 });1077 1078 data.grammar_triggers = {1079 { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },1080 { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },1081 { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },1082 { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }1083 };1084 }1085 1086 return data;1087}1088 1089static common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,1090 const autoparser::generation_params & inputs) {1091 common_chat_params data;1092 1093 data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);1094 1095 if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {1096 // This may happen if the model generates content + tool_call, the1097 // template does not add the model's next turn and confuses the model1098 // from emitting its proper reasoning token sequence.1099 data.prompt += "<|turn>model\n";1100 }1101 1102 data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;1103 data.supports_thinking = true;1104 data.thinking_start_tag = "<|channel>thought";1105 data.thinking_end_tag = "<channel|>";1106 1107 data.preserved_tokens = {1108 "<|channel>",1109 "<channel|>",1110 "<|tool_call>",1111 "<tool_call|>",1112 "<|turn>",1113 };1114 1115 auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();1116 auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();1117 auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);1118 auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;1119 1120 auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {1121 auto start = p.rule("start", p.prefix(inputs.generation_prompt, "<|channel>"));1122 1123 if (extract_reasoning) {1124 p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));1125 } else {1126 p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));1127 }1128 1129 auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");1130 auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));1131 1132 if (has_response_format) {1133 auto response_format = p.literal("```json") <<1134 p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<1135 p.literal("```");1136 return start + p.optional(thought) + response_format;1137 }1138 1139 if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {1140 // Gemma4 tool calling syntax1141 // Rules should match traversal logic in gemma4_to_json()1142 p.rule("gemma4-string-content", p.until("<|\"|>"));1143 p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));1144 p.rule("gemma4-bool", p.json_bool());1145 p.rule("gemma4-null", p.json_null());1146 p.rule("gemma4-number", p.json_number());1147 p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));1148 p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));1149 p.rule("gemma4-dict", [&]() {1150 auto ws = p.space();1151 auto member = p.ref("gemma4-dict-kv");1152 auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});1153 return p.sequence({1154 p.literal("{"), ws,1155 p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})1156 });1157 });1158 p.rule("gemma4-array", [&]() {1159 auto ws = p.space();1160 auto value = p.ref("gemma4-value");1161 auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});1162 return p.sequence({1163 p.literal("["), ws,1164 p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})1165 });1166 });1167 p.rule("gemma4-value", [&]() {1168 return p.choice({1169 p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),1170 p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")1171 });1172 });1173 1174 auto tool_choice = p.choice();1175 1176 foreach_function(inputs.tools, [&](const json & tool) {1177 const auto & function = tool.at("function");1178 std::string name = function.at("name");1179 // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules1180 // const auto & params = function.at("parameters");1181 1182 tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({1183 p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),1184 p.tool_args(p.ref("gemma4-dict")),1185 })));1186 });1187 1188 auto tool_call = p.trigger_rule("tool-call", p.repeat(1189 "<|tool_call>call:" + tool_choice + "<tool_call|>",1190 /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,1191 /* max = */ inputs.parallel_tool_calls ? -1 : 11192 ));1193 1194 auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));1195 auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));1196 auto message = p.rule("message", thought + content);1197 return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;1198 }1199 1200 // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may