Brunobkr/llama.cpp_AlgMor24_github

1#include "chat-auto-parser.h"2#include "chat-auto-parser-helpers.h"3#include "chat.h"4#include "log.h"5#include "jinja/caps.h"6#include "jinja/runtime.h"7 8#include <fstream>9#include <sstream>10#include <string>11#include <vector>12#include <algorithm>13 14#include "nlohmann/json.hpp"15 16using json = nlohmann::ordered_json;17 18// ANSI color codes - using 256-color palette for brighter colors (all bold)19#define ANSI_RESET       "\033[0m"20#define ANSI_PURPLE      "\033[1m\x1b[38;5;126m"  // Bold bright purple for main headers21#define ANSI_CYAN        "\033[1m\x1b[38;5;81m"   // Bold bright cyan for section headers22#define ANSI_BLUE        "\033[1m\x1b[38;5;12m"   // Bold bright blue for labels23#define ANSI_ORANGE      "\033[1m\x1b[38;5;209m"  // Bold orange for right differences24#define ANSI_GREEN       "\033[1m\x1b[38;5;83m"   // Bold bright green for left differences25#define ANSI_GRAY        "\033[1m\x1b[38;5;240m"  // Bold gray (used for "no variables" message)26#define ANSI_BOLD        "\033[1m"                // Standalone bold27#define ANSI_PREFIX      "\033[1m\x1b[38;5;176m"  // Bold color for common prefix28#define ANSI_SUFFIX      "\033[1m\x1b[38;5;61m"   // Bold color for common suffix29 30// All template paths extracted from tests/test-chat.cpp31static const std::vector<std::string> ALL_TEMPLATE_PATHS = {32    "models/templates/Apertus-8B-Instruct.jinja",33    "models/templates/Apriel-1.6-15b-Thinker-fixed.jinja",34    "models/templates/ByteDance-Seed-OSS.jinja",35    "models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja",36    "models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja",37    "models/templates/GLM-4.6.jinja",38    "models/templates/GLM-4.7-Flash.jinja",39    "models/templates/Kimi-K2-Instruct.jinja",40    "models/templates/Kimi-K2-Thinking.jinja",41    "models/templates/MiMo-VL.jinja",42    "models/templates/MiniMax-M2.jinja",43    "models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja",44    "models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja",45    "models/templates/NVIDIA-Nemotron-Nano-v2.jinja",46    "models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja",47    "models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja",48    "models/templates/Qwen-QwQ-32B.jinja",49    "models/templates/Qwen-Qwen2.5-7B-Instruct.jinja",50    "models/templates/Qwen3-Coder.jinja",51    "models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja",52    "models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja",53    "models/templates/deepseek-ai-DeepSeek-V3.1.jinja",54    "models/templates/fireworks-ai-llama-3-firefunction-v2.jinja",55    "models/templates/google-gemma-2-2b-it.jinja",56    "models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja",57    "models/templates/llama-cpp-deepseek-r1.jinja",58    "models/templates/meetkai-functionary-medium-v3.1.jinja",59    "models/templates/meetkai-functionary-medium-v3.2.jinja",60    "models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja",61    "models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja",62    "models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja",63    "models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja",64    "models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja",65    "models/templates/moonshotai-Kimi-K2.jinja",66    "models/templates/openai-gpt-oss-120b.jinja",67    "models/templates/unsloth-Apriel-1.5.jinja",68    "models/templates/unsloth-mistral-Devstral-Small-2507.jinja",69};70 71struct analysis_options {72    std::vector<std::string> template_paths;73    bool                     analyze_all = false;74};75 76static std::string read_file(const std::string & path) {77    std::ifstream fin(path, std::ios::binary);78    if (!fin.is_open()) {79        throw std::runtime_error("Could not open file: " + path);80    }81    std::ostringstream buf;82    buf << fin.rdbuf();83    return buf.str();84}85 86static void print_usage(const char * program_name) {87    LOG_ERR("Usage: %s [options]\n", program_name);88    LOG_ERR("\nOptions:\n");89    LOG_ERR("  --template <name>       Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n");90    LOG_ERR("  --template-file <path>  Analyze custom template file\n");91    LOG_ERR("  --all                   Analyze all templates from test suite\n");92    LOG_ERR("\nExamples:\n");93    LOG_ERR("  %s --all\n", program_name);94    LOG_ERR("  %s --template deepseek\n", program_name);95    LOG_ERR("  %s --template-file my-template.jinja\n", program_name);96}97 98static bool parse_options(int argc, char ** argv, analysis_options & opts) {99    if (argc < 2) {100        print_usage(argv[0]);101        return false;102    }103 104    for (int i = 1; i < argc; ++i) {105        std::string arg = argv[i];106 107        if (arg == "--all") {108            opts.analyze_all = true;109        } else if (arg == "--template") {110            if (i + 1 >= argc) {111                LOG_ERR("--template requires an argument\n");112                return false;113            }114            std::string pattern = argv[++i];115            std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);116 117            // Find matching templates118            bool found = false;119            for (const auto & path : ALL_TEMPLATE_PATHS) {120                std::string path_lower = path;121                std::transform(path_lower.begin(), path_lower.end(), path_lower.begin(), ::tolower);122                if (path_lower.find(pattern) != std::string::npos) {123                    opts.template_paths.push_back(path);124                    found = true;125                }126            }127 128            if (!found) {129                LOG_ERR("No templates found matching: %s\n", pattern.c_str());130                return false;131            }132        } else if (arg == "--template-file") {133            if (i + 1 >= argc) {134                LOG_ERR("--template-file requires an argument\n");135                return false;136            }137            opts.template_paths.push_back(argv[++i]);138        } else {139            LOG_ERR("Unknown option: %s\n", arg.c_str());140            print_usage(argv[0]);141            return false;142        }143    }144 145    if (opts.analyze_all) {146        opts.template_paths = ALL_TEMPLATE_PATHS;147    }148 149    if (opts.template_paths.empty()) {150        LOG_ERR("No templates specified\n");151        print_usage(argv[0]);152        return false;153    }154 155    return true;156}157 158static json build_tools_definition() {159    json parameters_schema                    = json::object();160    parameters_schema["type"]                 = "object";161    parameters_schema["properties"]           = json::object();162    parameters_schema["properties"]["param1"] = json::object({163        { "type",        "string"          },164        { "description", "First parameter" }165    });166    parameters_schema["properties"]["param2"] = json::object({167        { "type",        "string"           },168        { "description", "Second parameter" }169    });170    parameters_schema["required"]             = json::array({ "param1", "param2" });171 172    return json::array({173        json{ { "type", "function" },174             { "function", json{ { "name", "test_function_name" },175                                  { "description", "A test function for debugging" },176                                  { "parameters", parameters_schema } } } }177    });178}179 180// Helper to create a tool call with arguments as JSON object181static json build_tool_call(const std::string & name, const json & args_object, const std::string & id = "call_001") {182    return json{183        {"id", id},184        {"type", "function"},185        {"function", json{186            {"name", name},187            {"arguments", args_object}  // Pass as JSON object, not serialized string188        }}189    };190}191 192// Helper functions to create repeating message definitions193static json make_user_msg() {194    return json{195        {"role", "user"},196        {"content", "Hello, please help me."}197    };198}199 200static json make_user_msg2() {201    return json{202        {"role", "user"},203        {"content", "Thank you."}204    };205}206 207static json make_user_msg2_continue() {208    return json{209        {"role", "user"},210        {"content", "Continue."}211    };212}213 214static json make_assistant_no_tool() {215    return json{216        {"role", "assistant"},217        {"content", "Let me help you."}218    };219}220 221static json make_assistant_one_tool() {222    return json{223        {"role", "assistant"},224        {"content", nullptr},225        {"tool_calls", json::array({226            build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))227        })}228    };229}230 231static json make_assistant_two_tools() {232    return json{233        {"role", "assistant"},234        {"content", nullptr},235        {"tool_calls", json::array({236            build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})),237            build_tool_call("test_function_name", json::object({{"param1", "value3"}, {"param2", "value4"}}), "call_002")238        })}239    };240}241 242static json make_assistant_no_reasoning() {243    return json{244        {"role", "assistant"},245        {"content", "I can help you with that."}246    };247}248 249static json make_assistant_with_reasoning() {250    return json{251        {"role", "assistant"},252        {"content", "I can help you with that."},253        {"reasoning_content", "The user is asking for help. I should respond positively."}254    };255}256 257static json make_assistant_one_tool_with_reasoning() {258    return json{259        {"role", "assistant"},260        {"content", nullptr},261        {"tool_calls", json::array({262            build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))263        })},264        {"reasoning_content", "I need to call the tool first."}265    };266}267 268static void print_diff_split(const std::string & title, const diff_split & diff) {269    LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET);270    LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str());271    LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str());272    LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str());273    LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str());274}275 276static void check_reasoning_variables(const common_chat_template & tmpl) {277    LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET);278 279    try {280        // Create a list of candidate reasoning/thinking variable names to probe281        std::vector<std::string> candidate_vars = {282            "enable_reasoning",283            "use_reasoning",284            "reasoning_enabled",285            "has_reasoning",286            "reasoning_mode",287            "reasoning_format",288            "reasoning_active",289            "with_reasoning",290            "use_thinking",291            "thinking_enabled",292            "has_thinking",293            "thinking_mode",294            "thinking_format",295            "thinking_active",296            "with_thinking",297            "enable_reason",298            "reason_enabled",299            "enable_think",300            "think_enabled",301        };302 303        jinja::context ctx;304        ctx.is_get_stats = true;305 306        json messages = json::array({307            json{308                {"role", "user"},309                {"content", "Test message"}310            },311            json{312                {"role", "assistant"},313                {"content", "Response"},314                {"reasoning_content", "Some reasoning"}315            }316        });317 318        // Set up base context319        jinja::global_from_json(ctx, json{320            {"messages", messages},321            {"tools", json::array()},322            {"bos_token", ""},323            {"eos_token", ""},324            {"add_generation_prompt", false},325            {"enable_thinking", true}  // Already passed, so we'll exclude this from results326        }, true);327 328        // Add candidate variables as undefined to probe which ones are accessed329        for (const auto & var_name : candidate_vars) {330            ctx.set_val(var_name, jinja::mk_val<jinja::value_undefined_t>(var_name));331        }332 333        try {334            jinja::runtime runtime(ctx);335            runtime.execute(tmpl.prog);336        } catch (const std::exception & e) {337            // Execution may fail, that's okay - we just want to see what variables were accessed338        }339 340        // Check which candidate variables were accessed (stats.used = true)341        std::vector<std::string> accessed_vars;342        for (const auto & var_name : candidate_vars) {343            auto val = ctx.get_val(var_name);344            if (!val->is_undefined()) {345                // Variable was overwritten, skip it346                continue;347            }348            if (val->stats.used) {349                accessed_vars.push_back(var_name);350            }351        }352 353        if (accessed_vars.empty()) {354            LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET);355        } else {356            LOG_ERR("Template queries the following reasoning/thinking-related variables:\n");357            for (const auto & var : accessed_vars) {358                LOG_ERR("  %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET);359            }360        }361 362    } catch (const std::exception & e) {363        LOG_ERR("Error checking reasoning variables: %s\n", e.what());364    }365}366 367static void analyze_template(const std::string & template_path) {368    LOG_ERR("\n");369    LOG_ERR("%s", ANSI_PURPLE);370    LOG_ERR("================================================================================\n");371    LOG_ERR("                    ANALYZING TEMPLATE: %s\n", template_path.c_str());372    LOG_ERR("================================================================================\n");373    LOG_ERR("%s", ANSI_RESET);374 375    std::string template_source;376    try {377        template_source = read_file(template_path);378    } catch (const std::exception & e) {379        LOG_ERR("Error reading template: %s\n", e.what());380        return;381    }382 383    try {384        common_chat_template chat_template(template_source, "", "");385        json tools = build_tools_definition();386 387        // ===== CAPABILITIES ANALYSIS =====388        LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET);389        auto caps = chat_template.original_caps();390        LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false");391        LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false");392        LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false");393        LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false");394        LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false");395        LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false");396 397        // ===== DIFFERENTIAL ANALYSIS =====398 399        // Test 1: With and without tools (single user message)400        {401            json user_msg = make_user_msg();402 403            autoparser::generation_params params_no_tools;404            params_no_tools.messages = json::array({ user_msg });405            params_no_tools.add_generation_prompt = false;406            params_no_tools.tools = json::array();407 408            autoparser::generation_params params_with_tools = params_no_tools;409            params_with_tools.tools = tools;410 411            std::string output_no_tools = common_chat_template_direct_apply(chat_template, params_no_tools);412            std::string output_with_tools = common_chat_template_direct_apply(chat_template, params_with_tools);413 414            auto diff = calculate_diff_split(output_no_tools, output_with_tools);415            print_diff_split("Diff: With vs Without Tools (single user message)", diff);416        }417 418        // Test 2: With and without add_generation_prompt (single user message)419        {420            json user_msg = make_user_msg();421 422            autoparser::generation_params params_no_prompt;423            params_no_prompt.messages = json::array({ user_msg });424            params_no_prompt.add_generation_prompt = false;425            params_no_prompt.tools = json::array();426 427            autoparser::generation_params params_with_prompt = params_no_prompt;428            params_with_prompt.add_generation_prompt = true;429 430            std::string output_no_prompt = common_chat_template_direct_apply(chat_template, params_no_prompt);431            std::string output_with_prompt = common_chat_template_direct_apply(chat_template, params_with_prompt);432 433            auto diff = calculate_diff_split(output_no_prompt, output_with_prompt);434            print_diff_split("Diff: With vs Without add_generation_prompt (single user message)", diff);435        }436 437        // Test 3: Assistant with reasoning_content (user, assistant)438        {439            json user_msg = make_user_msg();440 441            autoparser::generation_params params_no_reasoning;442            params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning() });443            params_no_reasoning.add_generation_prompt = false;444            params_no_reasoning.enable_thinking = true;445 446            autoparser::generation_params params_with_reasoning = params_no_reasoning;447            params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning() });448 449            std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);450            std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);451 452            auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);453            print_diff_split("Diff: With vs Without reasoning_content (user, assistant)", diff);454        }455 456        // Test 4: Assistant with reasoning_content (user, assistant, user)457        {458            json user_msg = make_user_msg();459            json user_msg2 = make_user_msg2();460 461            autoparser::generation_params params_no_reasoning;462            params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning(), user_msg2 });463            params_no_reasoning.add_generation_prompt = false;464            params_no_reasoning.enable_thinking = true;465 466            autoparser::generation_params params_with_reasoning = params_no_reasoning;467            params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning(), user_msg2 });468 469            std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);470            std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);471 472            auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);473            print_diff_split("Diff: With vs Without reasoning_content (user, assistant, user)", diff);474        }475 476        // Test 5: Tool call in last assistant message (user, assistant)477        {478            json user_msg = make_user_msg();479 480            autoparser::generation_params params_no_tool;481            params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool() });482            params_no_tool.add_generation_prompt = false;483            params_no_tool.tools = tools;484 485            autoparser::generation_params params_with_tool = params_no_tool;486            params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool() });487 488            std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool);489            std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool);490 491            auto diff = calculate_diff_split(output_no_tool, output_with_tool);492            print_diff_split("Diff: With vs Without tool call (user, assistant)", diff);493        }494 495        // Test 6: Tool call in last assistant message (user, assistant, user)496        {497            json user_msg = make_user_msg();498            json user_msg2 = make_user_msg2_continue();499 500            autoparser::generation_params params_no_tool;501            params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool(), user_msg2 });502            params_no_tool.add_generation_prompt = false;503            params_no_tool.tools = tools;504 505            autoparser::generation_params params_with_tool = params_no_tool;506            params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });507 508            std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool);509            std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool);510 511            auto diff = calculate_diff_split(output_no_tool, output_with_tool);512            print_diff_split("Diff: With vs Without tool call (user, assistant, user)", diff);513        }514 515        // Test 7: One vs two tool calls (user, assistant)516        {517            json user_msg = make_user_msg();518 519            autoparser::generation_params params_one_tool;520            params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool() });521            params_one_tool.add_generation_prompt = false;522            params_one_tool.tools = tools;523 524            autoparser::generation_params params_two_tools = params_one_tool;525            params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools() });526 527            std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool);528            std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools);529 530            auto diff = calculate_diff_split(output_one_tool, output_two_tools);531            print_diff_split("Diff: One vs Two tool calls (user, assistant)", diff);532        }533 534        // Test 8: One vs two tool calls (user, assistant, user)535        {536            json user_msg = make_user_msg();537            json user_msg2 = make_user_msg2_continue();538 539            autoparser::generation_params params_one_tool;540            params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });541            params_one_tool.add_generation_prompt = false;542            params_one_tool.tools = tools;543 544            autoparser::generation_params params_two_tools = params_one_tool;545            params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools(), user_msg2 });546 547            std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool);548            std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools);549 550            auto diff = calculate_diff_split(output_one_tool, output_two_tools);551            print_diff_split("Diff: One vs Two tool calls (user, assistant, user)", diff);552        }553 554        // Test 9: Tool call with vs without reasoning_content (user, assistant)555        {556            json user_msg = make_user_msg();557 558            autoparser::generation_params params_no_reasoning;559            params_no_reasoning.messages = json::array({ user_msg, make_assistant_one_tool() });560            params_no_reasoning.add_generation_prompt = false;561            params_no_reasoning.tools = tools;562            params_no_reasoning.enable_thinking = true;563 564            autoparser::generation_params params_with_reasoning = params_no_reasoning;565            params_with_reasoning.messages = json::array({ user_msg, make_assistant_one_tool_with_reasoning() });566 567            std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);568            std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);569 570            auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);571            print_diff_split("Diff: Tool call with vs without reasoning_content (user, assistant)", diff);572        }573 574        // Check reasoning variables575        check_reasoning_variables(chat_template);576 577    } catch (const std::exception & e) {578        LOG_ERR("Analysis failed: %s\n", e.what());579    }580}581 582int main(int argc, char ** argv) {583    // Set log level to capture all output584    common_log_set_verbosity_thold(99);585 586    analysis_options opts;587    if (!parse_options(argc, argv, opts)) {588        return 1;589    }590 591    LOG_ERR("\n");592    LOG_ERR("%s", ANSI_PURPLE);593    LOG_ERR("================================================================================\n");594    LOG_ERR("                      TEMPLATE ANALYSIS TOOL\n");595    LOG_ERR("================================================================================\n");596    LOG_ERR("%s", ANSI_RESET);597    LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET);598 599    for (const auto & path : opts.template_paths) {600        analyze_template(path);601    }602 603    LOG_ERR("\n");604    LOG_ERR("%s", ANSI_GREEN);605    LOG_ERR("================================================================================\n");606    LOG_ERR("                      ANALYSIS COMPLETE\n");607    LOG_ERR("================================================================================\n");608    LOG_ERR("%s", ANSI_RESET);609 610    return 0;611}612