Brunobkr/llama.cpp_AlgMor24_github
ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.
03k
1#include "chat.h"2 3#include "chat-auto-parser-helpers.h"4#include "chat-auto-parser.h"5#include "chat-peg-parser.h"6#include "common.h"7#include "ggml.h"8#include "json-schema-to-grammar.h"9#include "log.h"10 11#include "jinja/value.h"12#include "jinja/runtime.h"13#include "jinja/caps.h"14#include "peg-parser.h"15 16#include "nlohmann/json.hpp"17 18#include <algorithm>19#include <cstdio>20#include <cstdlib>21#include <ctime>22#include <exception>23#include <functional>24#include <map>25 26#include <optional>27#include <sstream>28#include <stdexcept>29#include <string>30#include <utility>31#include <vector>32 33using json = nlohmann::ordered_json;34 35static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {36 auto time = std::chrono::system_clock::to_time_t(now);37 auto local_time = *std::localtime(&time);38 std::ostringstream ss;39 ss << std::put_time(&local_time, format.c_str());40 auto res = ss.str();41 return res;42}43 44static json safe_args_parse(const std::string & to_parse) {45 std::string stripped = to_parse;46 if (to_parse.at(0) == '"' && to_parse.at(to_parse.length() - 1) == '"') {47 stripped = to_parse.substr(1, to_parse.length() - 1);48 }49 try {50 return json::parse(stripped);51 } catch (json::exception & e) {52 return stripped;53 }54}55 56static std::string string_diff(const std::string & last, const std::string & current) {57 if (last.empty()) {58 return current;59 }60 if (!string_starts_with(current, last)) {61 if (string_starts_with(last, current)) {62 // This happens if the last generation ended on a partial stop word (not erased),63 // and the current ended on a stop word (erased).64 return "";65 }66 throw std::runtime_error("Invalid diff: '" + last + "' not found at start of '" + current + "'");67 }68 return current.substr(last.size());69}70 71static bool has_content_or_tool_calls(const common_chat_msg & msg) {72 return !msg.content.empty() || !msg.tool_calls.empty();73}74 75std::string common_chat_msg::render_content(const std::string & delimiter) const {76 if (!content.empty() && !content_parts.empty()) {77 throw std::runtime_error("Cannot specify both content and content_parts");78 }79 if (!content.empty()) {80 return content;81 }82 83 std::string text;84 for (const auto & part : content_parts) {85 if (part.type == "text") {86 if (!text.empty()) {87 text += delimiter;88 }89 text += part.text;90 }91 }92 return text;93}94 95common_chat_role common_chat_role_from_string(const std::string & role) {96 if (role == "system") { return COMMON_CHAT_ROLE_SYSTEM; }97 if (role == "assistant") { return COMMON_CHAT_ROLE_ASSISTANT; }98 if (role == "user") { return COMMON_CHAT_ROLE_USER; }99 if (role == "tool") { return COMMON_CHAT_ROLE_TOOL; }100 return COMMON_CHAT_ROLE_UNKNOWN;101}102 103const char * common_chat_role_to_string(common_chat_role role) {104 switch (role) {105 case COMMON_CHAT_ROLE_SYSTEM: return "system";106 case COMMON_CHAT_ROLE_ASSISTANT: return "assistant";107 case COMMON_CHAT_ROLE_USER: return "user";108 case COMMON_CHAT_ROLE_TOOL: return "tool";109 case COMMON_CHAT_ROLE_UNKNOWN: return "";110 }111 return "";112}113 114json common_chat_msg_delimiters::to_json() const {115 json result = json::array();116 for (const auto & d : delimiters) {117 result.push_back({118 { "role", common_chat_role_to_string(d.role) },119 { "delimiter", d.delimiter },120 });121 }122 return result;123}124 125common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {126 common_chat_msg_delimiters result;127 128 if (!delimiters.is_array()) {129 return result;130 }131 132 result.delimiters.reserve(delimiters.size());133 for (const auto & d : delimiters) {134 if (!d.is_object()) {135 continue;136 }137 result.delimiters.push_back({138 common_chat_role_from_string(d.value("role", std::string())),139 d.value("delimiter", std::string()),140 });141 }142 143 return result;144}145 146void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {147 for (auto & d : delimiters) {148 d.tokens = common_tokenize(vocab, d.delimiter, false, true);149 }150}151 152common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips) const {153 std::vector<std::pair<common_chat_role, size_t>> matches;154 155 auto skip = skips.begin();156 for (size_t i = 0; i < tokens.size();) {157 if (skip != skips.end() && i == skip->first) {158 i += skip->second;159 ++skip;160 continue;161 }162 for (const auto & d : delimiters) {163 if (i + d.tokens.size() > tokens.size()) {164 continue;165 }166 if (std::equal(d.tokens.begin(), d.tokens.end(), tokens.begin() + i)) {167 matches.emplace_back(d.role, i);168 break;169 }170 }171 i++;172 }173 174 matches.emplace_back(COMMON_CHAT_ROLE_UNKNOWN, tokens.size());175 176 common_chat_msg_spans spans;177 for (size_t i = 0; i + 1 < matches.size(); i++) {178 const auto & curr = matches[i];179 const auto & next = matches[i + 1];180 spans.add(curr.first, curr.second, next.second - curr.second);181 }182 183 return spans;184}185 186json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {187 if (!content.empty() && !content_parts.empty()) {188 throw std::runtime_error("Cannot specify both content and content_parts");189 }190 json jmsg {191 {"role", role},192 };193 if (!content.empty()) {194 jmsg["content"] = content;195 } else if (!content_parts.empty()) {196 if (concat_typed_text || contains_media()) {197 std::string text;198 bool last_was_media_marker = false;199 // join parts with newline, do not add newline before or after media markers200 for (const auto & part : content_parts) {201 bool add_new_line = true;202 if (part.type == "text") {203 add_new_line = !last_was_media_marker && !text.empty();204 last_was_media_marker = false;205 } else if (part.type == "media_marker") {206 add_new_line = false;207 last_was_media_marker = true;208 } else {209 LOG_WRN("Ignoring content part type: %s\n", part.type.c_str());210 continue;211 }212 213 if (add_new_line) {214 text += '\n';215 }216 217 text += part.text;218 }219 jmsg["content"] = text;220 } else {221 auto & parts = jmsg["content"] = json::array();222 for (const auto & part : content_parts) {223 parts.push_back({224 {"type", part.type},225 {"text", part.text},226 });227 }228 }229 } else {230 jmsg["content"] = "";231 }232 if (!reasoning_content.empty()) {233 jmsg["reasoning_content"] = reasoning_content;234 }235 if (!tool_name.empty()) {236 jmsg["name"] = tool_name;237 }238 if (!tool_call_id.empty()) {239 jmsg["tool_call_id"] = tool_call_id;240 }241 if (!tool_calls.empty()) {242 jmsg["tool_calls"] = json::array();243 auto & jtool_calls = jmsg["tool_calls"];244 for (const auto & tool_call : tool_calls) {245 json tc {246 {"type", "function"},247 {"function", {248 {"name", tool_call.name},249 {"arguments", json(tool_call.arguments)},250 }},251 };252 if (!tool_call.id.empty()) {253 tc["id"] = tool_call.id;254 }255 // Some templates generate and require an id (sometimes in a very specific format, e.g. Mistral Nemo).256 // We only generate a random id for the ones that don't generate one by themselves257 // (they also won't get to see it as their template likely doesn't use it, so it's all for the client)258 // {"id", tc.id.empty() ? gen_tool_call_id() : tc.id},259 jtool_calls.push_back(tc);260 }261 }262 263 return jmsg;264}265 266std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const common_chat_msg & msg_prv,267 const common_chat_msg & msg_new) {268 std::vector<common_chat_msg_diff> diffs;269 if (msg_new.tool_calls.size() > msg_prv.tool_calls.size()) {270 diffs.reserve(msg_new.tool_calls.size() - msg_prv.tool_calls.size() + 3);271 } else {272 diffs.reserve(3);273 }274 275 // TODO: these can become expensive for long messages - how to optimize?276 if (msg_prv.reasoning_content != msg_new.reasoning_content) {277 auto & diff = diffs.emplace_back();278 diff.reasoning_content_delta = string_diff(msg_prv.reasoning_content, msg_new.reasoning_content);279 }280 if (msg_prv.content != msg_new.content) {281 auto & diff = diffs.emplace_back();282 diff.content_delta = string_diff(msg_prv.content, msg_new.content);283 }284 285 if (msg_new.tool_calls.size() < msg_prv.tool_calls.size()) {286 std::string err = "Invalid diff: now finding less tool calls!\n";287 err += " Previous (" + std::to_string(msg_prv.tool_calls.size()) + "):\n";288 for (const auto & tc : msg_prv.tool_calls) {289 err += " - name: '" + tc.name + "', args: '" + tc.arguments + "'\n";290 }291 err += " Current (" + std::to_string(msg_new.tool_calls.size()) + "):\n";292 for (const auto & tc : msg_new.tool_calls) {293 err += " - name: '" + tc.name + "', args: '" + tc.arguments + "'\n";294 }295 err += " Current msg text content:\n" + msg_new.content + "\n";296 throw std::runtime_error(err);297 }298 299 if (!msg_prv.tool_calls.empty()) {300 const auto idx = msg_prv.tool_calls.size() - 1;301 const auto & pref = msg_prv.tool_calls[idx];302 const auto & newf = msg_new.tool_calls[idx];303 // Allow tool name to change during incremental parsing:304 // - empty -> non-empty (initial discovery)305 // - prefix -> longer string (name grows as more input is parsed)306 if (pref.name != newf.name && !pref.name.empty() && !newf.name.empty()) {307 // Check if one is a prefix of the other (for incremental parsing where names grow or shrink)308 bool is_prefix = (newf.name.rfind(pref.name, 0) == 0);309 if (!is_prefix) {310 LOG_ERR("Tool call mismatch: prev='%s' new='%s'\n", pref.name.c_str(), newf.name.c_str());311 throw std::runtime_error("Invalid diff: tool call mismatch!");312 }313 }314 const auto args_diff = string_diff(pref.arguments, newf.arguments);315 if (!args_diff.empty() || pref.id != newf.id || pref.name != newf.name) {316 auto & diff = diffs.emplace_back();317 diff.tool_call_index = idx;318 if (pref.id != newf.id || pref.name != newf.name) {319 diff.tool_call_delta.id = newf.id;320 diff.tool_call_delta.name = newf.name;321 }322 diff.tool_call_delta.arguments = args_diff;323 }324 }325 for (size_t idx = msg_prv.tool_calls.size(); idx < msg_new.tool_calls.size(); ++idx) {326 auto & diff = diffs.emplace_back();327 diff.tool_call_index = idx;328 diff.tool_call_delta = msg_new.tool_calls[idx];329 }330 331 return diffs;332}333 334using chat_template_caps = jinja::caps;335 336struct common_chat_templates {337 bool add_bos;338 bool add_eos;339 bool has_explicit_template; // Model had builtin template or template overridden was specified.340 std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)341 std::unique_ptr<common_chat_template> template_tool_use;342};343 344common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {345 if (tool_choice == "auto") {346 return COMMON_CHAT_TOOL_CHOICE_AUTO;347 }348 if (tool_choice == "none") {349 return COMMON_CHAT_TOOL_CHOICE_NONE;350 }351 if (tool_choice == "required") {352 return COMMON_CHAT_TOOL_CHOICE_REQUIRED;353 }354 throw std::invalid_argument("Invalid tool_choice: " + tool_choice);355}356 357bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates) {358 common_chat_templates_inputs inputs;359 inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;360 common_chat_msg msg;361 msg.role = "user";362 msg.content = "test";363 inputs.messages = { msg };364 inputs.enable_thinking = true;365 inputs.add_generation_prompt = true;366 inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;367 368 auto params = common_chat_templates_apply(chat_templates, inputs);369 return params.supports_thinking;370}371 372std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messages) {373 std::vector<common_chat_msg> msgs;374 375 try {376 if (!messages.is_array()) {377 throw std::invalid_argument("Expected 'messages' to be an array, got " + messages.dump());378 }379 380 for (const auto & message : messages) {381 if (!message.is_object()) {382 throw std::invalid_argument("Expected 'message' to be an object, got " + message.dump());383 }384 385 common_chat_msg msg;386 if (!message.contains("role")) {387 throw std::invalid_argument("Missing 'role' in message: " + message.dump());388 }389 msg.role = message.at("role");390 391 auto has_content = message.contains("content");392 auto has_tool_calls = message.contains("tool_calls");393 if (has_content) {394 const auto & content = message.at("content");395 if (content.is_string()) {396 msg.content = content;397 } else if (content.is_array()) {398 for (const auto & part : content) {399 if (!part.contains("type")) {400 throw std::invalid_argument("Missing content part type: " + part.dump());401 }402 const auto & type = part.at("type");403 if (type != "text" && type != "media_marker") {404 throw std::invalid_argument("Unsupported content part type: " + type.dump());405 }406 common_chat_msg_content_part msg_part;407 msg_part.type = type;408 msg_part.text = part.at("text");409 msg.content_parts.push_back(msg_part);410 }411 } else if (!content.is_null()) {412 throw std::invalid_argument("Invalid 'content' type: expected string or array, got " +413 content.dump() +414 " (ref: https://github.com/ggml-org/llama.cpp/issues/8367)");415 }416 }417 if (has_tool_calls) {418 for (const auto & tool_call : message.at("tool_calls")) {419 common_chat_tool_call tc;420 if (!tool_call.contains("type")) {421 throw std::invalid_argument("Missing tool call type: " + tool_call.dump());422 }423 const auto & type = tool_call.at("type");424 if (type != "function") {425 throw std::invalid_argument("Unsupported tool call type: " + tool_call.dump());426 }427 if (!tool_call.contains("function")) {428 throw std::invalid_argument("Missing tool call function: " + tool_call.dump());429 }430 const auto & fc = tool_call.at("function");431 if (!fc.contains("name")) {432 throw std::invalid_argument("Missing tool call name: " + tool_call.dump());433 }434 tc.name = fc.at("name");435 const auto & args = fc.at("arguments");436 if (args.is_string()) {437 tc.arguments = args;438 } else {439 tc.arguments = args.dump();440 }441 if (tool_call.contains("id")) {442 tc.id = tool_call.at("id");443 }444 msg.tool_calls.push_back(tc);445 }446 }447 if (!has_content && !has_tool_calls) {448 throw std::invalid_argument(449 "Expected 'content' or 'tool_calls' (ref: https://github.com/ggml-org/llama.cpp/issues/8367 & "450 "https://github.com/ggml-org/llama.cpp/issues/12279)");451 }452 if (message.contains("reasoning_content")) {453 msg.reasoning_content = message.at("reasoning_content");454 }455 if (message.contains("name")) {456 msg.tool_name = message.at("name");457 }458 if (message.contains("tool_call_id")) {459 msg.tool_call_id = message.at("tool_call_id");460 }461 462 msgs.push_back(msg);463 }464 } catch (const std::exception & e) {465 // @ngxson : disable otherwise it's bloating the API response466 // printf("%s\n", std::string("; messages = ") + messages.dump(2));467 throw std::runtime_error("Failed to parse messages: " + std::string(e.what()));468 }469 470 return msgs;471}472 473static json render_message_to_json(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {474 if (!c.supports_string_content && !c.supports_typed_content) {475 LOG_WRN("%s: Neither string content nor typed content is supported by the template. This is unexpected and may lead to issues.\n", __func__);476 }477 478 bool only_string_accepted = c.supports_string_content && !c.supports_typed_content;479 bool only_typed_accepted = !c.supports_string_content && c.supports_typed_content;480 481 json messages = json::array();482 for (const auto & msg : msgs) {483 if (only_string_accepted) {484 json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ true);485 messages.push_back(jmsg);486 } else if (only_typed_accepted) {487 json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);488 if (jmsg.at("content").is_string()) {489 jmsg["content"] = json::array({490 json{491 {"type", "text"},492 {"text", jmsg.at("content").get<std::string>()},493 }494 });495 }496 messages.push_back(jmsg);497 } else {498 json jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);499 messages.push_back(jmsg);500 }501 }502 return messages;503}504 505// DEPRECATED: only used in tests506json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {507 jinja::caps c;508 c.supports_string_content = true;509 c.supports_typed_content = !concat_typed_text;510 return render_message_to_json(msgs, c);511}512 513json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools) {514 if (tools.empty()) {515 return json();516 }517 518 auto result = json::array();519 for (const auto & tool : tools) {520 result.push_back({521 { "type", "function" },522 { "function", {523 { "name", tool.name },524 { "description", tool.description },525 { "parameters", json::parse(tool.parameters) },526 }},527 });528 }529 return result;530}531 532std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & tools) {533 std::vector<common_chat_tool> result;534 535 try {536 if (!tools.is_null()) {537 if (!tools.is_array()) {538 throw std::invalid_argument("Expected 'tools' to be an array, got " + tools.dump());539 }540 for (const auto & tool : tools) {541 if (!tool.contains("type")) {542 throw std::invalid_argument("Missing tool type: " + tool.dump());543 }544 const auto & type = tool.at("type");545 if (!type.is_string() || type != "function") {546 throw std::invalid_argument("Unsupported tool type: " + tool.dump());547 }548 if (!tool.contains("function")) {549 throw std::invalid_argument("Missing tool function: " + tool.dump());550 }551 552 const auto & function = tool.at("function");553 result.push_back({554 /* .name = */ function.at("name"),555 /* .description = */ function.value("description", ""),556 /* .parameters = */ function.value("parameters", json::object()).dump(),557 });558 }559 }560 } catch (const std::exception & e) {561 throw std::runtime_error("Failed to parse tools: " + std::string(e.what()) + "; tools = " + tools.dump(2));562 }563 564 return result;565}566 567common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value) {568 if (value.is_boolean() && value.get<bool>()) {569 return COMMON_CHAT_CONTINUATION_AUTO;570 }571 if (value.is_string()) {572 auto value_str = value.get<std::string>();573 if (value_str == "reasoning_content") {574 return COMMON_CHAT_CONTINUATION_REASONING;575 }576 if (value_str == "content") {577 return COMMON_CHAT_CONTINUATION_CONTENT;578 }579 }580 return COMMON_CHAT_CONTINUATION_NONE;581}582 583bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {584 if (use_jinja) {585 try {586 common_chat_msg msg;587 msg.role = "user";588 msg.content = "test";589 590 auto tmpls = common_chat_templates_init(/* model= */ nullptr, tmpl);591 592 common_chat_templates_inputs inputs;593 inputs.messages = { msg };594 595 common_chat_templates_apply(tmpls.get(), inputs);596 return true;597 } catch (const std::exception & e) {598 LOG_ERR("%s: failed to apply template: %s\n", __func__, e.what());599 return false;600 }601 }602 llama_chat_message chat[] = {603 { "user", "test" }604 };605 const int res = llama_chat_apply_template(tmpl.c_str(), chat, 1, true, nullptr, 0);606 return res >= 0;607}608 609std::string common_chat_format_single(const struct common_chat_templates * tmpls,610 const std::vector<common_chat_msg> & past_msg,611 const common_chat_msg & new_msg,612 bool add_ass,613 bool use_jinja) {614 common_chat_templates_inputs inputs;615 inputs.use_jinja = use_jinja;616 inputs.add_bos = tmpls->add_bos;617 inputs.add_eos = tmpls->add_eos;618 619 std::string fmt_past_msg;620 if (!past_msg.empty()) {621 inputs.messages = past_msg;622 inputs.add_generation_prompt = false;623 fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt;624 }625 std::ostringstream ss;626 // if the past_msg ends with a newline, we must preserve it in the formatted version627 if (add_ass && !fmt_past_msg.empty() && fmt_past_msg.back() == '\n') {628 ss << "\n";629 };630 // format chat with new_msg631 inputs.messages.push_back(new_msg);632 inputs.add_generation_prompt = add_ass;633 auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt;634 // get the diff part635 ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());636 return ss.str();637}638 639std::string common_chat_format_example(const struct common_chat_templates * tmpls,640 bool use_jinja,641 const std::map<std::string, std::string> & chat_template_kwargs) {642 common_chat_templates_inputs inputs;643 inputs.use_jinja = use_jinja;644 inputs.add_bos = tmpls->add_bos;645 inputs.add_eos = tmpls->add_eos;646 inputs.chat_template_kwargs = chat_template_kwargs;647 auto add_simple_msg = [&](auto role, auto content) {648 common_chat_msg msg;649 msg.role = role;650 msg.content = content;651 inputs.messages.push_back(msg);652 };653 add_simple_msg("system", "You are a helpful assistant");654 add_simple_msg("user", "Hello");655 add_simple_msg("assistant", "Hi there");656 add_simple_msg("user", "How are you?");657 return common_chat_templates_apply(tmpls, inputs).prompt;658}659 660#define CHATML_TEMPLATE_SRC \661 "{%- for message in messages -%}\n" \662 " {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' -}}\n" \663 "{%- endfor -%}\n" \664 "{%- if add_generation_prompt -%}\n" \665 " {{- '<|im_start|>assistant\n' -}}\n" \666 "{%- endif -%}"667 668void common_chat_templates_free(struct common_chat_templates * tmpls) {669 delete tmpls;670}671 672bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) {673 return tmpls->has_explicit_template;674}675 676// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list677// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call678static bool is_lfm2_template(const std::string & src) {679 return src.find("<|tool_list_start|>") != std::string::npos &&680 src.find("<|tool_list_end|>") != std::string::npos;681}682 683common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {684 common_chat_prompt_preset asr_preset;685 asr_preset.system = "";686 asr_preset.user = "Transcribe audio to text";687 688 if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {689 asr_preset.system = "Perform ASR.";690 asr_preset.user = "";691 }692 693 return asr_preset;694}695 696std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {697 if (!variant.empty()) {698 if (variant == "tool_use") {699 if (tmpls->template_tool_use) {700 return tmpls->template_tool_use->source();701 }702 return "";703 }704 LOG_DBG("%s: unknown template variant: %s\n", __func__, variant.c_str());705 }706 return tmpls->template_default->source();707}708 709common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,710 const std::string & chat_template_override,711 const std::string & bos_token_override,712 const std::string & eos_token_override) {713 std::string default_template_src;714 std::string template_tool_use_src;715 716 bool has_explicit_template = !chat_template_override.empty();717 if (chat_template_override.empty()) {718 GGML_ASSERT(model != nullptr);719 const auto * str = llama_model_chat_template(model, /* name */ nullptr);720 if (str) {721 default_template_src = str;722 has_explicit_template = true;723 }724 str = llama_model_chat_template(model, /* name */ "tool_use");725 if (str) {726 template_tool_use_src = str;727 has_explicit_template = true;728 }729 } else {730 default_template_src = chat_template_override;731 }732 if (default_template_src.empty() || default_template_src == "chatml") {733 if (!template_tool_use_src.empty()) {734 default_template_src = template_tool_use_src;735 } else {736 default_template_src = CHATML_TEMPLATE_SRC;737 }738 }739 740 // TODO @ngxson : this is a temporary hack to prevent chat template from throwing an error741 // Ref: https://github.com/ggml-org/llama.cpp/pull/15230#issuecomment-3173959633742 if (default_template_src.find("<|channel|>") != std::string::npos743 // search for the error message and patch it744 && default_template_src.find("in message.content or") != std::string::npos) {745 string_replace_all(default_template_src,746 "{%- if \"<|channel|>analysis<|message|>\" in message.content or "747 "\"<|channel|>final<|message|>\" in message.content %}",748 "{%- if false %}");749 }750 751 // TODO @aldehir : this is a temporary fix, pending Minja changes752 // Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664753 if (default_template_src.find("[TOOL_CALLS]") != std::string::npos754 // search for the error message and patch it755 && default_template_src.find("if (message['content'] is none or") != std::string::npos) {756 string_replace_all(default_template_src,757 "{%- if (message['content'] is none or message['content'] == '' or "758 "message['content']|length == 0) and (message['tool_calls'] is not defined or "759 "message['tool_calls'] is none or message['tool_calls']|length == 0) %}",760 "{%- if false %}");761 }762 763 std::string token_bos = bos_token_override;764 std::string token_eos = eos_token_override;765 bool add_bos = false;766 bool add_eos = false;767 if (model) {768 const auto * vocab = llama_model_get_vocab(model);769 const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {770 if (token == LLAMA_TOKEN_NULL) {771 if (default_template_src.find(jinja_variable_name) != std::string::npos ||772 template_tool_use_src.find(jinja_variable_name) != std::string::npos) {773 LOG_WRN(774 "common_chat_templates_init: warning: vocab does not have a %s token, jinja template won't "775 "work as intended.\n",776 name);777 }778 return std::string();779 }780 return common_token_to_piece(vocab, token, true);781 };782 token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");783 token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");784 add_bos = llama_vocab_get_add_bos(vocab);785 add_eos = llama_vocab_get_add_eos(vocab);786 }787 common_chat_templates_ptr tmpls(new common_chat_templates());788 tmpls->has_explicit_template = has_explicit_template;789 tmpls->add_bos = add_bos;790 tmpls->add_eos = add_eos;791 try {792 tmpls->template_default = std::make_unique<common_chat_template>(default_template_src, token_bos, token_eos);793 } catch (const std::exception & e) {794 LOG_ERR("%s: error: %s\n", __func__, e.what());795 LOG_ERR("%s: failed to initialize chat template\n", __func__);796 LOG_ERR("%s: please consider disabling jinja via --no-jinja, or using another chat template\n", __func__);797 throw e;798 }799 if (!template_tool_use_src.empty()) {800 try {801 tmpls->template_tool_use = std::make_unique<common_chat_template>(template_tool_use_src, token_bos, token_eos);802 } catch (const std::exception & e) {803 LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());804 }805 }806 return tmpls;807}808 809const char * common_chat_format_name(common_chat_format format) {810 switch (format) {811 case COMMON_CHAT_FORMAT_CONTENT_ONLY:812 return "Content-only";813 case COMMON_CHAT_FORMAT_PEG_SIMPLE:814 return "peg-simple";815 case COMMON_CHAT_FORMAT_PEG_NATIVE:816 return "peg-native";817 case COMMON_CHAT_FORMAT_PEG_GEMMA4:818 return "peg-gemma4";819 case COMMON_CHAT_FORMAT_PEG_MINIMAX_M3:820 return "peg-minimax-m3";821 default:822 throw std::runtime_error("Unknown chat format");823 }824}825 826const char * common_reasoning_format_name(common_reasoning_format format) {827 switch (format) {828 case COMMON_REASONING_FORMAT_NONE:829 return "none";830 case COMMON_REASONING_FORMAT_AUTO:831 return "auto";832 case COMMON_REASONING_FORMAT_DEEPSEEK:833 return "deepseek";834 case COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY:835 return "deepseek-legacy";836 default:837 throw std::runtime_error("Unknown reasoning format");838 }839}840 841common_reasoning_format common_reasoning_format_from_name(const std::string & format) {842 if (format == "none") {843 return COMMON_REASONING_FORMAT_NONE;844 }845 if (format == "auto") {846 return COMMON_REASONING_FORMAT_AUTO;847 }848 if (format == "deepseek") {849 return COMMON_REASONING_FORMAT_DEEPSEEK;850 }851 if (format == "deepseek-legacy") {852 return COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY;853 }854 throw std::runtime_error("Unknown reasoning format: " + format);855}856 857static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {858 for (const auto & tool : tools) {859 if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {860 LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());861 continue;862 }863 fn(tool);864 }865}866 867static void foreach_parameter(const json & function,868 const std::function<void(const std::string &, const json &, bool)> & fn) {869 if (!function.contains("parameters") || !function.at("parameters").is_object()) {870 return;871 }872 const auto & params = function.at("parameters");873 if (!params.contains("properties") || !params.at("properties").is_object()) {874 return;875 }876 const auto & props = params.at("properties");877 std::set<std::string> required;878 if (params.contains("required") && params.at("required").is_array()) {879 params.at("required").get_to(required);880 }881 for (const auto & [name, prop] : props.items()) {882 bool is_required = (required.find(name) != required.end());883 fn(name, prop, is_required);884 }885}886 887static std::string common_chat_template_direct_apply_impl(888 const common_chat_template & tmpl,889 const autoparser::generation_params & inputs,890 const std::optional<json> & messages_override = std::nullopt,891 const std::optional<json> & tools_override = std::nullopt,892 const std::optional<json> & additional_context = std::nullopt) {893 jinja::context ctx(tmpl.source());894 895 nlohmann::ordered_json inp = nlohmann::ordered_json{896 {"messages", messages_override.has_value() ? *messages_override : inputs.messages},897 {"bos_token", tmpl.bos_token()},898 {"eos_token", tmpl.eos_token()},899 {"enable_thinking", inputs.enable_thinking},900 };901 if (tools_override.has_value() || !inputs.tools.empty()) {902 inp["tools"] = tools_override.has_value() ? *tools_override : inputs.tools;903 }904 if (inputs.extra_context.is_object()) {905 // TODO: do we need to merge, or replacing is fine?906 for (const auto & [k, v] : inputs.extra_context.items()) {907 inp[k] = v;908 }909 }910 if (additional_context.has_value()) {911 // TODO: merge properly instead of overwriting (matching old behavior)912 for (const auto & [k, v] : additional_context->items()) {913 inp[k] = v;914 }915 }916 if (inputs.add_generation_prompt) {917 inp["add_generation_prompt"] = true;918 }919 if (inp.contains("preserve_reasoning") && inp["preserve_reasoning"].is_boolean()) {920 bool enabled = inp["preserve_reasoning"].get<bool>();921 jinja::caps_apply_preserve_reasoning(ctx, enabled);922 }923 924 jinja::global_from_json(ctx, inp, inputs.mark_input);925 926 // render927 jinja::runtime runtime(ctx);928 const jinja::value results = runtime.execute(tmpl.prog);929 auto parts = jinja::runtime::gather_string_parts(results);930 931 std::string result = parts->as_string().str();932 933 // TODO: improve this later934 if (inputs.add_bos && string_starts_with(result, tmpl.bos_token())) {935 result = result.substr(tmpl.bos_token().size());936 }937 if (inputs.add_eos && string_ends_with(result, tmpl.eos_token())) {938 result = result.substr(0, result.size() - tmpl.eos_token().size());939 }940 return result;941}942 943std::string common_chat_template_direct_apply(944 const common_chat_template & tmpl,945 const autoparser::generation_params & inputs) {946 return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);947}948 949static std::string common_chat_template_generation_prompt_impl(950 const common_chat_template & tmpl,951 const autoparser::generation_params & inputs,952 const std::optional<json> & messages_override = std::nullopt,953 const std::optional<json> & tools_override = std::nullopt,954 const std::optional<json> & additional_context = std::nullopt) {955 956 auto adjusted_messages = messages_override ? *messages_override : inputs.messages;957 958 autoparser::generation_params params = inputs;959 params.add_generation_prompt = false;960 params.continue_final_message = COMMON_CHAT_CONTINUATION_NONE;961 std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, adjusted_messages, tools_override, additional_context);962 params.add_generation_prompt = true;963 std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, adjusted_messages, tools_override, additional_context);964 965 size_t prefix_len = 0;966 size_t min_size = std::min(no_gen_prompt.size(), gen_prompt.size());967 while (prefix_len < min_size && no_gen_prompt[prefix_len] == gen_prompt[prefix_len]) {968 prefix_len++;969 }970 return gen_prompt.substr(prefix_len);971}972 973std::string common_chat_template_generation_prompt(974 const common_chat_template & tmpl,975 const autoparser::generation_params & inputs) {976 return common_chat_template_generation_prompt_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);977}978 979static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,980 const autoparser::generation_params & inputs) {981 common_chat_params data;982 983 // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja984 auto adjusted_messages = json::array();985 for (const auto & msg : inputs.messages) {986 auto role = msg.value("role", "");987 if (role != "system" && role != "assistant") {988 // Only adjust system and assistant messages. Interestingly, the system message may contain thinking.989 adjusted_messages.push_back(msg);990 continue;991 }992 993 auto content = json::array();994 995 // If message contains `reasoning_content`, add it as a block of type `thinking`996 if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {997 content.push_back({998 { "type", "thinking" },999 { "thinking", msg.at("reasoning_content").get<std::string>() },1000 });1001 }1002 1003 // If message contains `content`, add it as a block of type `text`1004 if (msg.contains("content")) {1005 if (msg.at("content").is_string()) {1006 content.push_back({1007 { "type", "text" },1008 { "text", msg.at("content").get<std::string>() },1009 });1010 } else if (msg.at("content").is_array()) {1011 auto blocks = msg.at("content");1012 content.insert(content.end(), blocks.begin(), blocks.end());1013 }1014 }1015 1016 auto adjusted = msg;1017 adjusted["content"] = content;1018 adjusted.erase("reasoning_content");1019 adjusted_messages.push_back(adjusted);1020 }1021 1022 auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();1023 auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();1024 auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;1025 auto include_grammar = true;1026 1027 data.supports_thinking = true;1028 data.thinking_start_tag = "[THINK]";1029 data.thinking_end_tags = {"[/THINK]"};1030 data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);1031 data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);1032 data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;1033 data.preserved_tokens = {1034 "[THINK]",1035 "[/THINK]",1036 "[TOOL_CALLS]",1037 "[ARGS]",1038 };1039 1040 if (inputs.has_continuation()) {1041 const auto & msg = inputs.continue_msg;1042 1043 data.generation_prompt = "[THINK]" + msg.reasoning_content;1044 if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {1045 data.generation_prompt += "[/THINK]" + msg.render_content();1046 }1047 1048 data.prompt += data.generation_prompt;1049 }1050 1051 auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {1052 auto generation_prompt = p.eps();1053 auto reasoning =1054 extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();1055 1056 // Response format parser1057 if (has_response_format) {1058 // Ministral wants to emit json surrounded by code fences1059 return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");1060 }1061 1062 // Tool call parser1063 if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {1064 auto tool_choice = p.choice();1065 foreach_function(inputs.tools, [&](const json & tool) {1066 const auto & function = tool.at("function");1067 std::string name = function.at("name");1068 const auto & schema = function.at("parameters");1069 1070 tool_choice |=1071 p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +1072 p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));1073 });1074 1075 auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;1076 auto max_calls = inputs.parallel_tool_calls ? -1 : 1;1077 auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));1078 1079 return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);1080 }1081 1082 // Content only parser1083 include_grammar = false;1084 return generation_prompt + (reasoning << p.content(p.rest()));1085 });1086 1087 data.parser = parser.save();1088 1089 if (include_grammar) {1090 data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;1091 1092 data.grammar = build_grammar([&](const common_grammar_builder & builder) {1093 foreach_function(inputs.tools, [&](const json & tool) {1094 const auto & function = tool.at("function");1095 auto schema = function.at("parameters");1096 builder.resolve_refs(schema);1097 });1098 if (has_response_format) {1099 auto schema = inputs.json_schema;1100 builder.resolve_refs(schema);1101 }1102 parser.build_grammar(builder, data.grammar_lazy);1103 });1104 1105 data.grammar_triggers = {1106 { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }1107 };1108 }1109 1110 return data;1111}1112 1113static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,1114 const autoparser::generation_params & inputs) {1115 common_chat_params data;1116 1117 const std::string GEN_PREFIX = "<|im_start|>assistant\n";1118 1119 data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);1120 data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);1121 data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;1122 1123 auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;1124 1125 data.supports_thinking = supports_reasoning;1126 data.preserved_tokens = {1127 "<tool_call>",1128 "</tool_call>",1129 };1130 1131 if (supports_reasoning) {1132 data.thinking_start_tag = "<think>";1133 // Support both </think> and <tool_call> as reasoning end sequences.1134 // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model1135 data.thinking_end_tags = { "</think>", "<tool_call>" };1136 data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });1137 }1138 1139 data.message_delimiters = {1140 { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },1141 { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 31142 { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash1143 { COMMON_CHAT_ROLE_USER, "<|im_start|>user" },1144 { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },1145 };1146 1147 auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();1148 auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();1149 auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;1150 auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);1151 1152 if (inputs.has_continuation()) {1153 const auto & msg = inputs.continue_msg;1154 1155 data.generation_prompt = GEN_PREFIX;1156 if (supports_reasoning) {1157 data.generation_prompt += "<think>\n" + msg.reasoning_content;1158 if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {1159 data.generation_prompt += "\n</think>\n\n";1160 }1161 }1162 if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {1163 data.generation_prompt += msg.render_content();1164 }1165 1166 data.prompt += data.generation_prompt;1167 }1168 1169 auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {1170 auto generation_prompt = p.literal(GEN_PREFIX);1171 1172 auto reasoning = p.eps();1173 if (supports_reasoning && extract_reasoning) {1174 reasoning = p.optional("<think>" + p.space() +1175 p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +1176 (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));1177 }1178 1179 // Response format parser1180 if (has_response_format) {1181 return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));1182 }1183 1184 // Tool call parser1185 if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {1186 auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));1187 auto arg_string = p.rule("xml-arg-string",1188 p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));1189 1190 auto tool_choice = p.choice();1191 foreach_function(inputs.tools, [&](const json & tool) {1192 const auto & function = tool.at("function");1193 std::string name = function.at("name");1194 auto parameters = function.contains("parameters") ? function.at("parameters") : json::object();1195 1196 auto schema_info = common_schema_info();1197 schema_info.resolve_refs(parameters);1198 1199 std::vector<common_peg_parser> required_args;1200 std::vector<common_peg_parser> optional_args;