Felipe97/llama-cpp-compiled

1#include "parsers.h"2 3namespace workaround {4 5// Gemma4 uses a custom tool_responses field instead of role:tool messages.6//7// This will transform a sequence of messages:8//   assistant(tool_call+) -> tool+ -> assistant(content)9//10// Into a single assistant message containing a tool_responses field:11//   assistant(content + tool_call + tool_responses)12//13// This is necessary for the Gemma4 chat template to properly format the prompt.14// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma415struct gemma4_model_turn_builder {16    json & messages;17    size_t pos;18    json tool_calls = json::array();19    json tool_responses = json::array();20    json content;21    json reasoning_content;22 23    gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}24 25    void collect() {26        // Collect the first assistant message27        auto & msg = messages[pos];28        if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {29            // According to the prompt formatting guide, we need to preserve reasoning_content30            // between function calls. The current chat templates do not support this, but we will do it anyway.31            reasoning_content = msg.at("reasoning_content");32        }33        for (auto & tc : msg.at("tool_calls")) {34            tool_calls.push_back(tc);35        }36        pos++;37 38        // Collect tool call results39        while (pos < messages.size() && messages[pos].value("role", "") == "tool") {40            collect_result(messages[pos]);41            pos++;42        }43 44        // Check if the next assistant message is the final message45        if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {46            auto & next = messages[pos];47            if (!has_tool_calls(next) && has_content(next)) {48                content = next.at("content");49                pos++;50            }51        }52    }53 54    void collect_result(const json & curr) {55        json response;56        if (curr.contains("content")) {57            const auto & content = curr.at("content");58            if (content.is_string()) {59                // Try to parse the content as JSON; fall back to raw string60                try {61                    response = json::parse(content.get<std::string>());62                } catch (...) {63                    response = content;64                }65            } else {66                response = content;67            }68        }69 70        std::string name;71 72        // Match name with corresponding tool call73        size_t idx = tool_responses.size();74        if (idx < tool_calls.size()) {75            auto & tc = tool_calls[idx];76            if (tc.contains("function")) {77                name = tc.at("function").value("name", "");78            }79        }80 81        // Fallback to the tool call id82        if (name.empty()) {83            name = curr.value("tool_call_id", "");84        }85 86        tool_responses.push_back({{"name", name}, {"response", response}});87    }88 89    json build() {90        collect();91 92        json msg = {93            {"role", "assistant"},94            {"tool_calls", tool_calls},95        };96        if (!tool_responses.empty()) {97            msg["tool_responses"] = tool_responses;98        }99        if (!content.is_null()) {100            msg["content"] = content;101        }102        if (!reasoning_content.is_null()) {103            msg["reasoning_content"] = reasoning_content;104        }105        return msg;106    }107 108    static bool has_content(const json & msg) {109        if (!msg.contains("content") || msg.at("content").is_null()) {110            return false;111        }112        const auto & content = msg.at("content");113        if (content.is_string() && !content.get<std::string>().empty()) {114            return true;115        }116        if (content.is_array() && !content.empty()) {117            return true;118        }119        return false;120    }121 122    static bool has_tool_calls(const json & msg) {123        return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();124    }125};126 127void convert_tool_responses_gemma4(json & messages) {128    json result = json::array();129    size_t i = 0;130 131    while (i < messages.size()) {132        auto & msg = messages[i];133 134        if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||135            !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {136            result.push_back(msg);137            i++;138            continue;139        }140 141        gemma4_model_turn_builder builder(messages, i);142        result.push_back(builder.build());143        i = builder.pos;144    }145 146    messages = result;147}148 149}150 151common_chat_params common_chat_params_init_gemma4(const common_chat_template &    tmpl,152                                                         const autoparser::generation_params & inputs) {153    common_chat_params data;154 155    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);156    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);157 158    if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {159        // This may happen if the model generates content + tool_call, the160        // template does not add the model's next turn and confuses the model161        // from emitting its proper reasoning token sequence.162        data.generation_prompt = "<|turn>model\n";163        data.prompt += data.generation_prompt;164    }165 166    data.message_delimiters = {167        { COMMON_CHAT_ROLE_USER,      "<|turn>user"  },168        { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" },169    };170 171    data.format            = COMMON_CHAT_FORMAT_PEG_GEMMA4;172    data.supports_thinking  = true;173    data.thinking_start_tag = "<|channel>thought";174    data.thinking_end_tags  = {"<channel|>"};175 176    data.preserved_tokens = {177        "<|channel>",178        "<channel|>",179        "<|tool_call>",180        "<tool_call|>",181        "<|turn>",182    };183 184    if (inputs.has_continuation()) {185        const auto & msg = inputs.continue_msg;186 187        data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : "";188        data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;189        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {190            data.generation_prompt += "<channel|>" + msg.render_content();191        }192 193        data.prompt += data.generation_prompt;194    }195 196    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();197    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();198    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);199    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;200 201    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {202        auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));203 204        if (extract_reasoning) {205            p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));206        } else {207            p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));208        }209 210        auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");211        auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));212 213        if (has_response_format) {214            auto response_format = p.literal("```json") <<215                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<216                p.literal("```");217            return start + p.optional(thought) + response_format;218        }219 220        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {221            // Gemma4 tool calling syntax222            // Rules should match traversal logic in gemma4_to_json()223            p.rule("gemma4-string-content", p.until("<|\"|>"));224            p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));225            p.rule("gemma4-bool", p.json_bool());226            p.rule("gemma4-null", p.json_null());227            p.rule("gemma4-number", p.json_number());228            p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));229            p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));230            p.rule("gemma4-dict", [&]() {231                auto ws = p.space();232                auto member = p.ref("gemma4-dict-kv");233                auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});234                return p.sequence({235                    p.literal("{"), ws,236                    p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})237                });238            });239            p.rule("gemma4-array", [&]() {240                auto ws = p.space();241                auto value = p.ref("gemma4-value");242                auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});243                return p.sequence({244                    p.literal("["), ws,245                    p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})246                });247            });248            p.rule("gemma4-value", [&]() {249                return p.choice({250                    p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),251                    p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")252                });253            });254 255            auto tool_choice = p.choice();256 257            foreach_function(inputs.tools, [&](const json & tool) {258                const auto & function = tool.at("function");259                std::string  name     = function.at("name");260                // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules261                // const auto & params   = function.at("parameters");262 263                tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({264                    p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),265                    p.tool_args(p.ref("gemma4-dict")),266                })));267            });268 269            auto tool_call = p.trigger_rule("tool-call", p.repeat(270                "<|tool_call>call:" + tool_choice + "<tool_call|>",271                /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,272                /* max = */ inputs.parallel_tool_calls ? -1 : 1273            ));274 275            auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));276            auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));277            auto message = p.rule("message", thought + content);278            return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;279        }280 281        // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may282        // also emit a single trailing <channel|> token. Consume all complete reasoning blocks and283        // then stop at the first unmatched <channel|> token.284        auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));285        auto message = p.rule("message", thought + content);286        return start + p.one_or_more(message);287    });288 289    data.parser = parser.save();290 291    if (include_grammar) {292        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));293        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {294            parser.build_grammar(builder, data.grammar_lazy);295        });296 297        data.grammar_triggers = {298            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },299        };300    }301 302    return data;303}304