echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0479
1// Tests chat handling, including grammar generation and parsing for tool calling, for various templates.2//3// Also acts as a CLI to generate a Markdown summary of the formats of Jinja templates,4// e.g. given Minja (http://github.com/google/minja) checked out in parent dir:5//6// cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null7//8#include "../src/llama-grammar.h"9#include "../src/unicode.h"10#include "chat-auto-parser.h"11#include "chat.h"12#include "common.h"13#include "ggml.h"14#include "log.h"15 16#include <algorithm>17#include <exception>18#include <fstream>19#include <functional>20#include <iostream>21#include <nlohmann/json.hpp>22#include <set>23#include <stdexcept>24#include <string>25 26using json = nlohmann::ordered_json;27 28static std::ostream & operator<<(std::ostream & os, const common_chat_msg_diff & diff) {29 os << "{ content_delta: " << diff.content_delta << "; ";30 os << "reasoning_content_delta: " << diff.reasoning_content_delta << "; ";31 if (diff.tool_call_index != std::string::npos) {32 os << "tool_call_index: " << diff.tool_call_index << "; ";33 os << "tool_call_delta.name: " << diff.tool_call_delta.name << "; ";34 os << "tool_call_delta.id: " << diff.tool_call_delta.id << "; ";35 os << "tool_call_delta.arguments: " << diff.tool_call_delta.arguments << "; ";36 }37 os << "}";38 return os;39}40 41// operator<< for vector<common_chat_msg_diff>:42static std::ostream & operator<<(std::ostream & os, const std::vector<common_chat_msg_diff> & diffs) {43 os << "[\n";44 for (const auto & diff : diffs) {45 os << " " << diff << ",\n";46 }47 os << "]";48 return os;49}50 51static std::ostream & operator<<(std::ostream & os, const common_chat_msg & msg) {52 os << "{ role: " << msg.role << "; ";53 os << "content: " << msg.content << "; ";54 os << "content_parts: [\n";55 for (const auto & part : msg.content_parts) {56 os << " { type: " << part.type << "; text: " << part.text << " },\n";57 }58 os << "]; ";59 os << "reasoning_content: " << msg.reasoning_content << "; ";60 os << "tool_calls: [\n";61 for (const auto & tool_call : msg.tool_calls) {62 os << " { name: " << tool_call.name << "; arguments: " << tool_call.arguments << "; id: " << tool_call.id63 << " },\n";64 }65 os << "]";66 os << "}";67 return os;68}69 70template <class T> static bool equals(const T & expected, const T & actual) {71 return expected == actual;72}73 74static common_chat_msg normalize(const common_chat_msg & msg) {75 common_chat_msg normalized = msg;76 for (auto & tool_call : normalized.tool_calls) {77 try {78 tool_call.arguments = json::parse(tool_call.arguments).dump();79 } catch (const std::exception &) {80 }81 }82 return normalized;83}84 85template <> bool equals(const common_chat_msg & expected, const common_chat_msg & actual) {86 return normalize(expected) == normalize(actual);87}88 89template <class T> static void assert_equals(const T & expected, const T & actual) {90 if (!equals(expected, actual)) {91 std::ostringstream oss_expected;92 oss_expected << expected;93 std::ostringstream oss_actual;94 oss_actual << actual;95 LOG_ERR("Expected: %s\n", oss_expected.str().c_str());96 LOG_ERR("Actual: %s\n", oss_actual.str().c_str());97 common_log_flush(common_log_main());98 throw std::runtime_error("Test failed");99 }100}101 102static std::string read_file(const std::string & path) {103 std::ifstream fs(path, std::ios_base::binary);104 if (!fs.is_open()) {105 fs = std::ifstream("../" + path, std::ios_base::binary);106 if (!fs.is_open()) {107 throw std::runtime_error("Failed to open file: " + path);108 }109 }110 fs.seekg(0, std::ios_base::end);111 auto size = fs.tellg();112 fs.seekg(0);113 std::string out;114 out.resize(static_cast<size_t>(size));115 fs.read(out.data(), static_cast<std::streamsize>(size));116 return out;117}118 119static common_chat_templates_ptr read_templates(const std::string & path) {120 return common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, read_file(path)));121}122 123static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) {124 return std::unique_ptr<llama_grammar>(125 llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0));126}127 128// Helper to format a code point as a readable string129static std::string format_codepoint(uint32_t cp) {130 if (cp >= 32 && cp < 127) {131 return std::string("'") + static_cast<char>(cp) + "'";132 } else if (cp == '\n') {133 return "'\\n'";134 } else if (cp == '\r') {135 return "'\\r'";136 } else if (cp == '\t') {137 return "'\\t'";138 } else {139 return "U+" + std::to_string(cp);140 }141}142 143// Helper to format expected element from grammar stack144static std::string format_expected_element(const llama_grammar_rules & /* rules*/, const llama_grammar_element * elem) {145 if (!elem) {146 return "<end>";147 }148 149 switch (elem->type) {150 case LLAMA_GRETYPE_END:151 return "<end of rule>";152 case LLAMA_GRETYPE_ALT:153 return "<alternative>";154 case LLAMA_GRETYPE_RULE_REF:155 {156 // Find rule name - just show rule ID for now157 return "<rule-" + std::to_string(elem->value) + ">";158 }159 case LLAMA_GRETYPE_CHAR:160 {161 std::string result;162 const llama_grammar_element * pos = elem;163 bool first = true;164 165 do {166 if (!first) {167 result += " | ";168 }169 first = false;170 171 if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {172 // Range like [a-z]173 result += "[" + format_codepoint(pos->value) + "-" + format_codepoint(pos[1].value) + "]";174 pos += 2;175 } else {176 result += format_codepoint(pos->value);177 pos += 1;178 }179 } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);180 181 return result;182 }183 case LLAMA_GRETYPE_CHAR_NOT:184 {185 std::string result = "[^";186 const llama_grammar_element * pos = elem;187 bool first = true;188 189 do {190 if (!first) {191 result += " ";192 }193 first = false;194 195 if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {196 result += format_codepoint(pos->value) + "-" + format_codepoint(pos[1].value);197 pos += 2;198 } else {199 result += format_codepoint(pos->value);200 pos += 1;201 }202 } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);203 204 return result + "]";205 }206 case LLAMA_GRETYPE_CHAR_ANY:207 return "<any char>";208 case LLAMA_GRETYPE_TOKEN:209 return "<token-" + std::to_string(elem->value) + ">";210 case LLAMA_GRETYPE_TOKEN_NOT:211 return "<not-token-" + std::to_string(elem->value) + ">";212 default:213 return "<unknown>";214 }215}216 217// Get description of what the grammar expects at current position218static std::string get_expected_description(const llama_grammar_rules & rules, const llama_grammar_stacks & stacks) {219 if (stacks.empty()) {220 return "<no valid continuations>";221 }222 223 std::string result;224 std::set<std::string> seen;225 226 for (const auto & stack : stacks) {227 if (stack.empty()) {228 if (seen.insert("<end>").second) {229 if (!result.empty()) {230 result += " OR ";231 }232 result += "<end>";233 }234 continue;235 }236 237 const llama_grammar_element * elem = stack.back();238 std::string desc = format_expected_element(rules, elem);239 if (seen.insert(desc).second) {240 if (!result.empty()) {241 result += " OR ";242 }243 result += desc;244 }245 }246 247 return result;248}249 250// Result of a detailed grammar match attempt251struct grammar_match_result {252 bool success = false; // Did the string fully match the grammar?253 size_t matched_bytes = 0; // Bytes successfully matched before failure254 size_t matched_codepoints = 0; // Codepoints successfully matched before failure255 size_t total_bytes = 0; // Total bytes in input256 size_t total_codepoints = 0; // Total codepoints in input257 std::string matched_prefix; // The portion that was successfully matched258 std::string failing_char; // The character that caused failure (if any)259 std::string expected_description; // What the grammar expected at failure point260 bool incomplete = false; // True if matched all input but grammar expects more261};262 263// Detailed version of match_string that returns failure information264static grammar_match_result match_string_detailed(const std::string & input, llama_grammar * grammar) {265 grammar_match_result result;266 result.total_bytes = input.size();267 268 const auto cpts = unicode_cpts_from_utf8(input);269 result.total_codepoints = cpts.size();270 271 auto & stacks_cur = llama_grammar_get_stacks(grammar);272 const auto & rules = llama_grammar_get_rules(grammar);273 274 size_t byte_pos = 0;275 276 for (size_t i = 0; i < cpts.size(); i++) {277 const auto & cpt = cpts[i];278 279 // Get expected before accepting (for error reporting)280 std::string expected_before = get_expected_description(rules, stacks_cur);281 282 llama_grammar_accept(grammar, cpt);283 284 // Calculate byte position for this codepoint285 size_t cpt_bytes = 0;286 if (cpt < 0x80) {287 cpt_bytes = 1;288 } else if (cpt < 0x800) {289 cpt_bytes = 2;290 } else if (cpt < 0x10000) {291 cpt_bytes = 3;292 } else {293 cpt_bytes = 4;294 }295 296 if (stacks_cur.empty()) {297 // Grammar failed to match at this point298 result.matched_bytes = byte_pos;299 result.matched_codepoints = i;300 result.matched_prefix = input.substr(0, byte_pos);301 result.failing_char = format_codepoint(cpt);302 result.expected_description = expected_before;303 result.incomplete = false;304 return result;305 }306 307 byte_pos += cpt_bytes;308 }309 310 // All input matched - check if grammar is complete311 result.matched_bytes = input.size();312 result.matched_codepoints = cpts.size();313 result.matched_prefix = input;314 315 if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) {316 // An empty stack means that the grammar has been completed317 result.success = true;318 result.incomplete = false;319 } else {320 // Grammar expects more input321 result.success = false;322 result.incomplete = true;323 result.expected_description = get_expected_description(rules, stacks_cur);324 }325 326 return result;327}328 329// TODO: extract to common helper (copied from test-grammar-integration.cpp)330static bool match_string(const std::string & input, llama_grammar * grammar) {331 const auto cpts = unicode_cpts_from_utf8(input);332 333 auto & stacks_cur = llama_grammar_get_stacks(grammar);334 335 for (const auto & cpt : cpts) {336 llama_grammar_accept(grammar, cpt);337 338 if (stacks_cur.empty()) {339 // no stacks means that the grammar failed to match at this point340 return false;341 }342 }343 344 if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) {345 // An empty stack means that the grammar has been completed346 return true;347 }348 349 return false;350}351 352static std::string renormalize_json(const std::string & json_str) {353 try {354 auto json_obj = json::parse(json_str);355 return json_obj.dump();356 } catch (const std::exception & e) {357 return ""; // ignore parial JSON contents for comparison purposes358 }359}360 361static void assert_msg_equals(const common_chat_msg & expected,362 const common_chat_msg & actual,363 bool ignore_whitespace_differences = false) {364 assert_equals(expected.role, actual.role);365 if (ignore_whitespace_differences) {366 assert_equals(string_strip(expected.content), string_strip(actual.content));367 } else {368 assert_equals(expected.content, actual.content);369 }370 assert_equals(expected.content_parts.size(), actual.content_parts.size());371 for (size_t i = 0; i < expected.content_parts.size(); i++) {372 const auto & expected_part = expected.content_parts[i];373 const auto & actual_part = actual.content_parts[i];374 assert_equals(expected_part.type, actual_part.type);375 if (ignore_whitespace_differences) {376 assert_equals(string_strip(expected_part.text), string_strip(actual_part.text));377 } else {378 assert_equals(expected_part.text, actual_part.text);379 }380 }381 if (ignore_whitespace_differences) {382 assert_equals(string_strip(expected.reasoning_content), string_strip(actual.reasoning_content));383 } else {384 assert_equals(expected.reasoning_content, actual.reasoning_content);385 }386 assert_equals(expected.tool_calls.size(), actual.tool_calls.size());387 for (size_t i = 0; i < expected.tool_calls.size(); i++) {388 const auto & expected_tool_call = expected.tool_calls[i];389 const auto & actual_tool_call = actual.tool_calls[i];390 assert_equals(expected_tool_call.name, actual_tool_call.name);391 assert_equals(renormalize_json(expected_tool_call.arguments), renormalize_json(actual_tool_call.arguments));392 assert_equals(expected_tool_call.id, actual_tool_call.id);393 }394}395 396static common_chat_tool special_function_tool{397 /* .name = */ "special_function",398 /* .description = */ "I'm special",399 /* .parameters = */ R"({400 "type": "object",401 "properties": {402 "arg1": {403 "type": "integer",404 "description": "The arg."405 }406 },407 "required": ["arg1"]408 })",409};410static common_chat_tool special_function_tool_with_optional_param{411 /* .name = */ "special_function_with_opt",412 /* .description = */ "I'm special but have optional stuff",413 /* .parameters = */ R"({414 "type": "object",415 "properties": {416 "arg1": {417 "type": "integer",418 "description": "The arg."419 },420 "arg2": {421 "type": "integer",422 "description": "The optional arg."423 }424 },425 "required": ["arg1"]426 })",427};428 429static common_chat_tool empty_args_tool{430 /* .name = */ "empty_args",431 /* .description = */ "A tool that takes no arguments",432 /* .parameters = */ R"({433 "type": "object",434 "properties": {}435 })",436};437 438static common_chat_tool empty_args_tool_no_properties{439 /* .name = */ "empty_args_no_props",440 /* .description = */ "A tool that takes no arguments and has no properties",441 /* .parameters = */ R"({442 "type": "object"443 })",444};445 446static common_chat_tool python_tool{447 /* .name = */ "python",448 /* .description = */ "an ipython interpreter",449 /* .parameters = */ R"({450 "type": "object",451 "properties": {452 "code": {453 "type": "string",454 "description": "Python code to execute."455 }456 },457 "required": ["code"]458 })",459};460 461static common_chat_tool html_tool{462 /* .name = */ "html",463 /* .description = */ "an html validator",464 /* .parameters = */ R"({465 "type": "object",466 "properties": {467 "markup": {468 "type": "string",469 "description": "HTML markup to validate."470 }471 },472 "required": ["markup"]473 })",474};475 476static common_chat_tool get_time_tool{477 /* .name = */ "get_time",478 /* .description = */ "Get the current time in a city",479 /* .parameters = */ R"({480 "type": "object",481 "properties": {482 "city": {483 "type": "string",484 "description": "City name"485 }486 },487 "required": ["city"]488 })",489};490 491static common_chat_tool get_weather_tool{492 /* .name = */ "get_weather",493 /* .description = */ "Get the current weather in a city",494 /* .parameters = */ R"({495 "type": "object",496 "properties": {497 "city": {498 "type": "string",499 "description": "City name"500 }501 },502 "required": ["city"]503 })",504};505 506static common_chat_tool todo_list{507 /* .name = */ "todo_list",508 /* .description = */ "Create or update the todo list",509 /* .parameters = */ R"({510 "type": "object",511 "properties": {512 "todos": {513 "type": "array",514 "description": "List of TODO list items"515 }516 },517 "required": ["todos"]518 })",519};520 521static common_chat_tool edit_tool{522 /* .name = */ "edit",523 /* .description = */ "Edit file",524 /* .parameters = */ R"({525 "type": "object",526 "properties": {527 "filename": {528 "type": "string",529 "description": "Path of file to edit"530 },531 "oldString": {532 "type": "string",533 "description": "String to replace"534 },535 "newString": {536 "type": "string",537 "description": "New (replacement) value"538 }539 },540 "required": ["filename", "oldString", "newString"]541 })",542};543 544static common_chat_tool magic_tool{545 /* .name = */ "magic",546 /* .description = */ "Magic tool that takes a hash",547 /* .parameters = */ R"({548 "type": "object",549 "properties": {550 "name": {551 "type": "string"552 },553 "ref": {554 "type": "string"555 }556 },557 "required": ["name", "ref"]558 })",559};560 561static common_chat_tool magic_int_tool{562 /* .name = */ "magic_int",563 /* .description = */ "Magic tool that takes a hash",564 /* .parameters = */ R"({565 "type": "object",566 "properties": {567 "ref": {568 "type": "integer"569 },570 "name": {571 "type": "string"572 }573 },574 "required": ["ref"]575 })",576};577 578static common_chat_tool amount_tool{579 /* .name = */ "amount",580 /* .description = */ "Amount converter",581 /* .parameters = */ R"({582 "type": "object",583 "properties": {584 "orig": {585 "type": "number"586 }587 },588 "required": ["orig"]589 })",590};591 592static common_chat_tool toggle_tool{593 /* .name = */ "toggle",594 /* .description = */ "Toggle a feature",595 /* .parameters = */ R"({596 "type": "object",597 "properties": {598 "enabled": {599 "type": "boolean",600 "description": "Whether to enable the feature"601 }602 },603 "required": ["enabled"]604 })",605};606 607static common_chat_tool nullable_tool{608 /* .name = */ "set_nullable",609 /* .description = */ "Set a nullable value",610 /* .parameters = */ R"({611 "type": "object",612 "properties": {613 "value": {614 "type": "null",615 "description": "A null value"616 }617 },618 "required": ["value"]619 })",620};621 622static common_chat_tool config_tool{623 /* .name = */ "set_config",624 /* .description = */ "Set configuration",625 /* .parameters = */ R"({626 "type": "object",627 "properties": {628 "config": {629 "type": "object",630 "description": "Configuration dict"631 }632 },633 "required": ["config"]634 })",635};636 637static common_chat_tool imaginary_number_tool{638 /* .name = */ "imaginary_number",639 /* .description = */ "Imaginary number converter",640 /* .parameters = */ R"({641 "type": "object",642 "properties": {643 "number": {644 "type": "object",645 "properties": {646 "real": {647 "type": "number"648 },649 "imaginary": {650 "type": "number"651 }652 },653 "required": ["real", "imaginary"]654 }655 },656 "required": ["number"]657 })",658};659 660static common_chat_tool nullable_string_tool{661 /* .name = */ "set_nullable_str",662 /* .description = */ "Set a nullable string value",663 /* .parameters = */ R"({664 "type": "object",665 "properties": {666 "name": {667 "type": ["string", "null"],668 "description": "A nullable string"669 }670 },671 "required": ["name"]672 })",673};674 675static common_chat_tool nullable_string_null_first_tool{676 /* .name = */ "set_nullable_str_nf",677 /* .description = */ "Set a nullable string value with null first in type array",678 /* .parameters = */ R"({679 "type": "object",680 "properties": {681 "name": {682 "type": ["null", "string"],683 "description": "A nullable string with null first"684 }685 },686 "required": ["name"]687 })",688};689 690static common_chat_tool nullable_int_tool{691 /* .name = */ "set_nullable_int",692 /* .description = */ "Set a nullable integer value",693 /* .parameters = */ R"({694 "type": "object",695 "properties": {696 "count": {697 "type": ["integer", "null"],698 "description": "A nullable integer"699 }700 },701 "required": ["count"]702 })",703};704 705static common_chat_tool enum_no_type_tool{706 /* .name = */ "set_unit",707 /* .description = */ "Set a temperature unit",708 /* .parameters = */ R"({709 "type": "object",710 "properties": {711 "unit": {712 "enum": ["celsius", "fahrenheit"],713 "description": "Temperature unit"714 }715 },716 "required": ["unit"]717 })",718};719 720static common_chat_tool string_param_tool{721 /* .name = */ "string_param",722 /* .description = */ "Tool with string parameter for testing",723 /* .parameters = */ R"({724 "type": "object",725 "properties": {726 "text": {727 "type": "string",728 "description": "A text parameter"729 }730 },731 "required": []732 })",733};734 735static common_chat_tool quoted_unquoted_tool{736 /* .name = */ "quoted_unquoted",737 /* .description = */ "Tool with two string parameters, one for quoted string, one for unquoted",738 /* .parameters = */ R"({739 "type": "object",740 "properties": {741 "quoted": {742 "type": "string",743 "description": "Quoted value"744 },745 "unquoted": {746 "type": "string",747 "description": "Unquoted value"748 }749 },750 "required": ["quoted", "unquoted"]751 })",752};753 754 755static common_chat_tool tool_2req_4opt{756 /* .name = */ "tool_2req_4opt",757 /* .description = */ "Tool with 2 required and 4 optional params",758 /* .parameters = */ R"({759 "type": "object",760 "properties": {761 "req1": { "type": "string", "description": "Required string" },762 "req2": { "type": "integer", "description": "Required int" },763 "opt1": { "type": "string", "description": "Optional string 1" },764 "opt2": { "type": "integer", "description": "Optional int 1" },765 "opt3": { "type": "string", "description": "Optional string 2" },766 "opt4": { "type": "integer", "description": "Optional int 2" }767 },768 "required": ["req1", "req2"]769 })",770};771 772static common_chat_tool tool_2req_5opt{773 /* .name = */ "tool_2req_5opt",774 /* .description = */ "Tool with 2 required and 5 optional params",775 /* .parameters = */ R"({776 "type": "object",777 "properties": {778 "req1": { "type": "string", "description": "Required string" },779 "req2": { "type": "integer", "description": "Required int" },780 "opt1": { "type": "string", "description": "Optional string 1" },781 "opt2": { "type": "integer", "description": "Optional int 1" },782 "opt3": { "type": "string", "description": "Optional string 2" },783 "opt4": { "type": "integer", "description": "Optional int 2" },784 "opt5": { "type": "string", "description": "Optional string 3" }785 },786 "required": ["req1", "req2"]787 })",788};789 790static std::vector<common_chat_tool> tools{ special_function_tool, special_function_tool_with_optional_param,791 python_tool, html_tool, todo_list };792 793const common_chat_msg message_user{794 "user",795 "Hey there!",796 /* .content_parts = */ {},797 /* .tool_calls = */ {},798 /* .reasoning_content = */ "",799 /* .tool_name = */ "",800 /* .tool_call_id = */ "",801};802 803const common_chat_msg message_user_parts{804 "user",805 /* .content = */ "",806 /* .content_parts = */807 {808 { "text", "Hey" },809 { "text", "there" },810 },811 /* .tool_calls = */812 { },813 /* .reasoning_content = */814 "",815 /* .tool_name = */ "",816 /* .tool_call_id = */ "",817};818 819static common_chat_msg simple_assist_msg(const std::string & content,820 const std::string & reasoning_content = "",821 const std::string & tool_name = "",822 const std::string & arguments = "",823 const std::string & id = "") {824 common_chat_msg msg;825 msg.role = "assistant";826 msg.content = content;827 msg.reasoning_content = reasoning_content;828 if (!tool_name.empty() || !id.empty()) {829 msg.tool_calls.push_back({ tool_name, arguments, id });830 }831 return msg;832}833 834static common_chat_msg message_with_tool_calls(const std::string & tool_name, const std::string & arguments) {835 return simple_assist_msg("", "", tool_name, arguments);836}837 838static common_chat_msg message_with_tool_calls_and_reasoning(const std::string & tool_name,839 const std::string & arguments,840 const std::string & reasoning) {841 return simple_assist_msg("", reasoning, tool_name, arguments);842}843 844static common_chat_msg message_with_reasoning_content_and_multiple_tool_calls(845 const std::string & reasoning,846 const std::string & content,847 const std::vector<std::pair<std::string, std::string>> & tool_calls) {848 common_chat_msg msg;849 msg.role = "assistant";850 msg.content = content;851 msg.reasoning_content = reasoning;852 for (const auto & [name, args] : tool_calls) {853 msg.tool_calls.push_back({ name, args, "" });854 }855 return msg;856}857 858static common_chat_msg message_with_content_and_tool_call(const std::string & content,859 const std::string & tool_name,860 const std::string & arguments) {861 return simple_assist_msg(content, "", tool_name, arguments);862}863 864static common_chat_msg message_with_reasoning_and_tool_call(const std::string & reasoning,865 const std::string & tool_name,866 const std::string & arguments) {867 return simple_assist_msg("", reasoning, tool_name, arguments);868}869 870const common_chat_msg message_assist = simple_assist_msg("Hello, world!\nWhat's up?");871const common_chat_msg message_assist_empty = simple_assist_msg("");872const common_chat_msg message_assist_thoughts_unparsed_deepseek =873 simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?");874const common_chat_msg message_assist_thoughts_unparsed_md =875 simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}```");876const common_chat_msg message_assist_thoughts_unparsed_md_partial =877 simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}");878 879const common_chat_msg message_assist_thoughts_unparsed_r7b =880 simple_assist_msg("<|START_THINKING|>I'm\nthinking<|END_THINKING|>Hello, world!\nWhat's up?");881const common_chat_msg message_assist_thoughts_unparsed_magistral =882 simple_assist_msg("[THINK]raisonnement[/THINK]Réponse");883const common_chat_msg message_assist_thoughts = simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking");884const common_chat_msg message_assist_thoughts_unopened_unparsed =885 simple_assist_msg("I'm\nthinking</think>Hello, world!\nWhat's up?");886const common_chat_msg message_assist_thoughts_no_content = simple_assist_msg("", "I'm\nthinking");887const common_chat_msg message_assist_call = simple_assist_msg("", "", "special_function", "{\"arg1\": 1}");888const common_chat_msg message_assist_call_noopt =889 simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1}");890const common_chat_msg message_assist_call_withopt =891 simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1, \"arg2\": 2}");892const common_chat_msg message_assist_call_content =893 simple_assist_msg("Hello, world!\nWhat's up?", "", "special_function", "{\"arg1\":1}");894const common_chat_msg message_assist_call_empty_args = simple_assist_msg("", "", "special_function");895const common_chat_msg message_assist_call_cutoff_args = simple_assist_msg("", "", "special_function", "{\"arg");896const common_chat_msg message_assist_call_thoughts =897 simple_assist_msg("", "I'm\nthinking", "special_function", "{\"arg1\":1}");898const common_chat_msg message_assist_call_thoughts_unparsed =899 simple_assist_msg("<think>I'm\nthinking</think>\n\n", "", "special_function", "{\"arg1\": 1}");900const common_chat_msg message_assist_call_thoughts_content =901 simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking", "special_function", "{\"arg1\": 1}");902const common_chat_msg message_assist_call_id =903 simple_assist_msg("", "", "special_function", "{\"arg1\":1}", /* .id = */ "123456789");904const common_chat_msg message_assist_call_idx =905 simple_assist_msg("", "", "special_function", "{\"arg1\":1}", /* .id = */ "0");906const common_chat_msg message_assist_thoughts_call_idx =907 simple_assist_msg("", "I'm\nthinking", "special_function", "{\"arg1\": 1}", /* id = */ "0");908const common_chat_msg message_assist_thoughts_partial_call =909 simple_assist_msg("", "I'm\nthinking", "special_function", "", /* id = */ "0");910const common_chat_msg message_assist_call_python = simple_assist_msg("", "", "python", "{\"code\":\"print('hey')\"}");911const common_chat_msg message_assist_call_python_lines =912 simple_assist_msg("", "", "python", "{\"code\":\"# This is a program:\\nprint('hey')\"}");913const common_chat_msg message_assist_call_python_lines_unclosed =914 simple_assist_msg("", "", "python", "{\"code\":\"# This is a program:\\nprint('hey')");915const common_chat_msg message_assist_json_content =916 simple_assist_msg("{\n \"response\": \"Hello, world!\\nWhat's up?\"\n}");917 918// Use for PEG parser implementations919struct peg_test_case {920 common_chat_templates_inputs params;921 std::string input;922 common_chat_msg expect;923 bool is_partial = false;924 bool expect_reconstruction = false;925};926 927struct make_peg_parser {928 common_chat_params params_;929 common_peg_arena arena_;930 bool detailed_debug_;931 932 make_peg_parser(common_chat_templates * tmpls,933 const common_chat_templates_inputs & inputs,934 bool detailed_debug = false) {935 detailed_debug_ = detailed_debug;936 params_ = common_chat_templates_apply(tmpls, inputs);937 arena_.load(params_.parser);938 }939 940 common_chat_msg parse(const std::string & msg, bool is_partial) const {941 common_chat_parser_params parser_params(params_);942 parser_params.debug = detailed_debug_;943 return common_chat_peg_parse(arena_, msg, is_partial, parser_params);944 }945};946 947// Global template filter for --template flag948static std::string g_template_filter;949 950// When true, run reconstruction test on every non-partial test and report results951static bool g_force_reconstruction_test = false;952 953static void test_peg_parser(common_chat_templates * tmpls,954 const std::function<void(peg_test_case &)> & init,955 bool detailed_debug) {956 // UTF-8-safe truncation helper (same as in test_parser_with_streaming)957 constexpr auto utf8_truncate_safe_len = [](const std::string_view s) -> size_t {958 auto len = s.size();959 if (len == 0) {960 return 0;961 }962 auto i = len;963 for (size_t back = 0; back < 4 && i > 0; ++back) {964 --i;965 unsigned char c = s[i];966 if ((c & 0x80) == 0) {967 return len;968 }969 if ((c & 0xC0) == 0xC0) {970 size_t expected_len = 0;971 if ((c & 0xE0) == 0xC0) {972 expected_len = 2;973 } else if ((c & 0xF0) == 0xE0) {974 expected_len = 3;975 } else if ((c & 0xF8) == 0xF0) {976 expected_len = 4;977 } else {978 return i;979 }980 if (len - i >= expected_len) {981 return len;982 }983 return i;984 }985 }986 return len - std::min(len, size_t(3));987 };988 989 peg_test_case tc;990 init(tc);991 if (tc.params.messages.empty()) {992 tc.params.messages = { message_user };993 }994 if (tc.expect.role.empty()) {995 tc.expect.role = "assistant";996 }997 998 auto parser = make_peg_parser(tmpls, tc.params, detailed_debug);999 if (detailed_debug) {1000 LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str());1001 LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str());1002 }1003 1004 common_chat_msg msg_accum;1005 common_chat_msg msg_prev;1006 msg_accum.role = msg_prev.role = "assistant";1007 1008 for (size_t i = 1; i <= tc.input.size(); ++i) {1009 auto is_partial = i < tc.input.size() || tc.is_partial;1010 // Use UTF-8 safe truncation to avoid corrupting multi-byte characters1011 size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));1012 std::string prefix = tc.input.substr(0, safe_len);1013 common_chat_msg msg_current = parser.parse(prefix, is_partial);1014 1015 for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) {1016 if (!diff.reasoning_content_delta.empty()) {1017 msg_accum.reasoning_content += diff.reasoning_content_delta;1018 }1019 if (!diff.content_delta.empty()) {1020 msg_accum.content += diff.content_delta;1021 }1022 if (diff.tool_call_index != std::string::npos) {1023 // During partial parsing, a new tool call may appear with empty name initially1024 // The name gets filled in as more input is parsed1025 while (msg_accum.tool_calls.size() <= diff.tool_call_index) {1026 msg_accum.tool_calls.push_back({ "", "", "" });1027 }1028 // Always update name and id from diff (may change during incremental parsing), but only if the delta1029 // actually contains them1030 if (!diff.tool_call_delta.name.empty()) {1031 msg_accum.tool_calls[diff.tool_call_index].name = diff.tool_call_delta.name;1032 }1033 if (!diff.tool_call_delta.id.empty()) {1034 msg_accum.tool_calls[diff.tool_call_index].id = diff.tool_call_delta.id;1035 }1036 if (!diff.tool_call_delta.arguments.empty()) {1037 msg_accum.tool_calls[diff.tool_call_index].arguments += diff.tool_call_delta.arguments;1038 }1039 }1040 }1041 try {1042 assert_msg_equals(msg_current, msg_accum, true);1043 } catch (std::exception & e) {1044 throw std::runtime_error((std::string("Error comparing accumulated message to current: ") + e.what()).c_str());1045 }1046 1047 msg_prev = msg_current;1048 }1049 1050 if (!tc.is_partial) {1051 assert_msg_equals(tc.expect, parser.parse(tc.input, false), true);1052 }1053 assert_msg_equals(tc.expect, msg_accum, true);1054 1055 // Test grammar if present in params1056 if (!parser.params_.grammar.empty()) {1057 auto grammar = build_grammar(parser.params_.grammar);1058 if (!grammar) {1059 throw std::runtime_error("Failed to build grammar: " + parser.params_.grammar);1060 }1061 1062 // In production, grammar triggers match against the full generated text1063 // including the generation prompt. All positions are in full_input coordinates.1064 const auto & gen_prompt = parser.params_.generation_prompt;1065 std::string full_input = gen_prompt + tc.input;1066 1067 // Determine whether the reasoning-budget sampler path applies: tool-call grammar1068 // with all WORD triggers and thinking tags present. In production, the reasoning1069 // budget sampler inhibits grammar application while inside thinking blocks —1070 // triggers inside <think>...</think> are suppressed.1071 bool use_reasoning_budget_path = false;1072 if (parser.params_.grammar_lazy && !parser.params_.thinking_end_tag.empty()) {1073 use_reasoning_budget_path = true;1074 for (const auto & trigger : parser.params_.grammar_triggers) {1075 if (trigger.type != COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {1076 use_reasoning_budget_path = false;1077 break;1078 }1079 }1080 }1081 1082 // Find the earliest trigger position to determine the constrained portion1083 auto earliest_trigger_pos = std::string::npos;1084 1085 if (use_reasoning_budget_path) {1086 // Reasoning-budget path: simulate thinking-aware trigger detection.1087 // Walk through full_input tracking thinking state; only match triggers1088 // when outside thinking blocks.1089 const auto & think_start = parser.params_.thinking_start_tag;1090 const auto & think_end = parser.params_.thinking_end_tag;1091 1092 bool in_thinking = false;1093 for (size_t i = 0; i < full_input.size(); ++i) {1094 if (!in_thinking && !think_start.empty()1095 && full_input.compare(i, think_start.size(), think_start) == 0) {1096 in_thinking = true;1097 i += think_start.size() - 1;1098 continue;1099 }1100 if (in_thinking && full_input.compare(i, think_end.size(), think_end) == 0) {1101 in_thinking = false;1102 i += think_end.size() - 1;1103 continue;1104 }1105 if (in_thinking) {1106 continue;1107 }1108 // Outside thinking — check if any trigger word starts here1109 for (const auto & trigger : parser.params_.grammar_triggers) {1110 if (full_input.compare(i, trigger.value.size(), trigger.value) == 0) {1111 if (earliest_trigger_pos == std::string::npos || i < earliest_trigger_pos) {1112 earliest_trigger_pos = i;1113 }1114 }1115 }1116 if (earliest_trigger_pos != std::string::npos) {1117 break; // found the earliest1118 }1119 }1120 1121 // If the reasoning-budget path found no trigger outside thinking but the test1122 // expects tool calls, this template nests tool calls inside thinking1123 // blocks (e.g. Kimi). Fall back to the legacy path for this case.1124 if (earliest_trigger_pos == std::string::npos && !tc.expect.tool_calls.empty()) {1125 use_reasoning_budget_path = false;1126 }1127 }1128 1129 if (!use_reasoning_budget_path) {1130 // Legacy path: find triggers without thinking-awareness1131 for (const auto & trigger : parser.params_.grammar_triggers) {1132 size_t pos = std::string::npos;1133 std::smatch match;1134 switch (trigger.type) {1135 case COMMON_GRAMMAR_TRIGGER_TYPE_WORD:1136 {1137 const auto & word = trigger.value;1138 pos = full_input.find(word);1139 break;1140 }1141 case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN:1142 {1143 const auto & compiled = std::regex(trigger.value);1144 if (std::regex_search(full_input, match, compiled)) {1145 pos = match.position(compiled.mark_count());1146 }1147 break;1148 }1149 case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL:1150 {1151 // In production, PATTERN_FULL triggers are checked against1152 // the text generated so far, growing token by token. Simulate1153 // by trying every prefix of full_input.1154 const auto & compiled = std::regex(trigger.value);1155 for (size_t end = gen_prompt.size(); end <= full_input.size(); ++end) {1156 std::string prefix = full_input.substr(0, end);1157 if (std::regex_match(prefix, match, compiled)) {1158 pos = std::string::npos;1159 for (size_t gi = 1; gi < match.size(); ++gi) {1160 if (match[gi].length() > 0) {1161 pos = match.position(gi);1162 break;1163 }1164 }1165 if (pos == std::string::npos) {1166 pos = match.position(0);1167 }1168 break;1169 }1170 }1171 break;1172 }1173 default:1174 throw std::runtime_error("Unknown trigger type");1175 }1176 if (pos != std::string::npos) {1177 if (earliest_trigger_pos == std::string::npos || pos < earliest_trigger_pos) {1178 earliest_trigger_pos = pos;1179 }1180 }1181 }1182 }1183 1184 // If the test expects tool calls and the grammar is lazy, the trigger must fire.1185 // Otherwise the grammar would never activate in production and tool calls wouldn't1186 // be constrained. A silent skip here would hide broken triggers.1187 if (parser.params_.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial1188 && earliest_trigger_pos == std::string::npos) {1189 std::string trigger_desc;1190 for (const auto & trigger : parser.params_.grammar_triggers) {1191 trigger_desc += "\n [type=" + std::to_string(trigger.type) + "] " + trigger.value;1192 }1193 throw std::runtime_error(1194 "Grammar trigger did not fire, but test expects tool calls (lazy grammar).\n"1195 ">>> Input: " + full_input + "\n"1196 ">>> Triggers (" + std::to_string(parser.params_.grammar_triggers.size()) + "):" + trigger_desc);1197 }1198 1199 // Determine the constrained portion of input to test against grammar.1200 // If the trigger position falls inside the generation prompt, the grammar