Brunobkr/llama.cpp_AlgMor24_github

1#include "chat-auto-parser-helpers.h"2#include "chat-auto-parser.h"3#include "chat-peg-parser.h"4#include "chat.h"5#include "common.h"6#include "json-schema-to-grammar.h"7#include "log.h"8#include "nlohmann/json.hpp"9#include "peg-parser.h"10 11#include <stdexcept>12#include <string>13 14using json = nlohmann::ordered_json;15 16// Helper to iterate over tools/functions17static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {18    for (const auto & tool : tools) {19        if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {20            continue;21        }22        fn(tool);23    }24}25 26namespace autoparser {27 28parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :29    p(p),30    inputs(inputs),31    reasoning_parser(p.eps()) {}32 33common_chat_params peg_generator::generate_parser(const common_chat_template &    tmpl,34                                                  const struct generation_params & inputs) {35    // Run differential analysis to extract template structure36    struct autoparser autoparser;37    autoparser.analyze_template(tmpl);38    return generate_parser(tmpl, inputs, autoparser);39}40 41common_chat_params peg_generator::generate_parser(const common_chat_template &    tmpl,42                                                  const struct generation_params & inputs,43                                                  const autoparser &              autoparser) {44    // Create the result structure45    common_chat_params data;46    data.prompt            = common_chat_template_direct_apply(tmpl, inputs);47    data.generation_prompt = common_chat_template_generation_prompt(tmpl, inputs);48    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;49    data.preserved_tokens  = autoparser.preserved_tokens;50    data.additional_stops.insert(data.additional_stops.end(),51        autoparser.additional_stops.begin(), autoparser.additional_stops.end());52 53    std::string parser_generation_prompt = data.generation_prompt;54 55    if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !inputs.continue_msg.empty()) {56        // Build up generation prompt manually57        const auto & msg = inputs.continue_msg;58 59        if (!autoparser.reasoning.start.empty()) {60            data.generation_prompt = data.generation_prompt.substr(0, data.generation_prompt.find(autoparser.reasoning.start));61            data.generation_prompt += autoparser.reasoning.start + msg.reasoning_content;62            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {63                data.generation_prompt += autoparser.reasoning.end;64            }65        }66 67        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {68            data.generation_prompt += msg.render_content();69        }70 71        data.prompt += data.generation_prompt;72    }73 74    auto parser = autoparser.build_parser(inputs, parser_generation_prompt);75    data.parser = parser.save();76 77    // Build grammar if tools are present78    bool has_tools =79        autoparser.tools.format.mode != tool_format::NONE && inputs.tools.is_array() && !inputs.tools.empty();80    std::string trigger_marker = !autoparser.tools.format.section_start.empty() ? autoparser.tools.format.section_start :81                                                                                  autoparser.tools.format.per_call_start;82 83    bool has_response_format = !inputs.json_schema.empty() && inputs.json_schema.is_object();84    bool include_grammar = has_response_format || (has_tools &&85            ((inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO && !trigger_marker.empty()) ||86              inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));87 88    if (include_grammar) {89        data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;90        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {91            foreach_function(inputs.tools, [&](const json & tool) {92                const auto & function = tool.at("function");93                auto         schema   = function.contains("parameters") ? function.at("parameters") : json::object();94                builder.resolve_refs(schema);95            });96            if (has_response_format) {97                auto schema = inputs.json_schema;98                builder.resolve_refs(schema);99            }100            parser.build_grammar(builder, data.grammar_lazy);101        });102 103        // Set grammar triggers based on tool section markers (fall back to per-call markers)104        if (data.grammar_lazy) {105            data.grammar_triggers = {106                { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker }107            };108            if (autoparser.tools.format.openai_wrapper_trigger) {109                // model emits the OpenAI function wrapper, trigger on it110                data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "{\"type\": \"function\"," });111            }112        }113    }114 115    return data;116}117 118common_peg_arena autoparser::build_parser(const generation_params & inputs, const std::string & generation_prompt) const {119    if (!analysis_complete) {120        throw std::invalid_argument("Cannot call build_parser on autoparser without performing analysis first, call analyze_template(...)");121    }122    return build_chat_peg_parser([&](common_chat_peg_builder & p) {123        parser_build_context ctx(p, inputs);124        bool                 extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;125 126        ctx.extracting_reasoning = extract_reasoning && reasoning.mode != reasoning_mode::NONE;127        ctx.content              = &content;128        ctx.reasoning            = &reasoning;129 130        // Build reasoning parser131        ctx.reasoning_parser = reasoning.build_parser(ctx);132 133        auto parser = p.eps();134 135        bool has_tools           = inputs.tools.is_array() && !inputs.tools.empty();136        bool has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();137        bool pure_content        = reasoning.mode == reasoning_mode::NONE;138 139        if (has_response_format) {140            auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));141            parser = ctx.reasoning_parser + p.space() + p.choice({142                p.literal("```json") + p.space() + response_format + p.space() + p.literal("```"),143                p.space() + response_format  + p.space()144            }) + p.end();145            pure_content = false;146        } else if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && jinja_caps.supports_tool_calls) {147            parser = tools.build_parser(ctx);148            pure_content = false;149        } else {150            parser = content.build_parser(ctx);151        }152        const std::string reasoning_start = trim_whitespace(reasoning.start);153        return pure_content ? p.prefix(generation_prompt, reasoning_start) + parser : p.prefix(generation_prompt, reasoning_start) << parser;154    });155}156 157common_peg_parser analyze_reasoning::build_parser(parser_build_context & ctx) const {158    auto & p = ctx.p;159 160    if (!ctx.extracting_reasoning) {161        return p.eps();162    }163 164    if (mode == reasoning_mode::TAG_BASED || mode == reasoning_mode::TOOLS_ONLY) {165        if (!end.empty()) {166            if (!start.empty()) {167                // Standard tag-based: optional(<think>reasoning</think>)168                return p.optional(p.optspace(start) + p.reasoning(p.until(trim_whitespace(end))) + p.optspace(end));169            }170            // Delimiter-style (empty start)171            return p.optional(p.reasoning(p.until(trim_whitespace(end))) + p.optspace(end));172        }173    }174 175    return p.eps();176}177 178common_peg_parser analyze_content::build_parser(parser_build_context & ctx) const {179    auto & p = ctx.p;180 181    if (is_always_wrapped()) {182        if (ctx.extracting_reasoning) {183            return ctx.reasoning_parser + start + p.content(p.until(end)) + end + p.end();184        }185        return p.content(p.until(start)) + start + p.content(p.until(end)) + end + p.end();186    }187    return ctx.reasoning_parser + p.content(p.rest()) + p.end();188}189 190common_peg_parser analyze_content::build_optional_wrapped(parser_build_context & ctx) const {191    auto & p = ctx.p;192 193    if (is_always_wrapped()) {194        return p.optional(start + p.content(p.until(end)) + end);195    }196    return p.eps();197}198 199common_peg_parser analyze_tools::build_parser(parser_build_context & ctx) const {200    switch (format.mode) {201        case tool_format::JSON_NATIVE:202            return build_tool_parser_json_native(ctx);203        case tool_format::TAG_WITH_JSON:204            return build_tool_parser_tag_json(ctx);205        case tool_format::TAG_WITH_TAGGED:206            return build_tool_parser_tag_tagged(ctx);207        default:208            LOG_ERR("[ERROR] Template seems to support tool calls, but failed to determine tool format. Tool calling will not work properly. "209                "Check for a fixed template for your model in the models/templates directory of your llama.cpp installation or "210                "report an issue at https://github.com/ggml-org/llama.cpp/issues\n");211            return ctx.p.eps();212    }213}214 215common_peg_parser analyze_tools::build_tool_parser_json_native(parser_build_context & ctx) const {216    auto &       p           = ctx.p;217    const auto & inputs      = ctx.inputs;218 219    // Build effective field names with dot notation if function_field is set220    std::string name_field = format.name_field;221    std::string args_field = format.args_field;222 223    if (!format.function_field.empty() && format.function_field != "function" &&224        name_field.find('.') == std::string::npos) {225        name_field = format.function_field + "." + name_field;226        args_field = format.function_field + "." + args_field;227    }228 229    auto tools_parser = p.eps();230    if (format.section_start.empty() && !format.per_call_start.empty()) {231        auto single_tool_parser = p.standard_json_tools(232            format.per_call_start, format.per_call_end, inputs.tools, inputs.parallel_tool_calls,233            inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,234            format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);235        tools_parser = p.trigger_rule("tool-calls", p.one_or_more(single_tool_parser + p.space()));236    } else {237        tools_parser = p.standard_json_tools(238            format.section_start, format.section_end, inputs.tools, inputs.parallel_tool_calls,239            inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,240            format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger);241    }242 243    // Handle content wrappers if present244    if (ctx.content && ctx.content->is_always_wrapped()) {245        auto wrapped_content = ctx.content->build_optional_wrapped(ctx);246        return ctx.reasoning_parser + wrapped_content + tools_parser + p.end();247    }248 249    std::string tool_start = "{";250    if (!format.section_start.empty()) {251        tool_start = format.section_start;252    } else if (!format.per_call_start.empty()) {253        tool_start = format.per_call_start;254    }255 256    return ctx.reasoning_parser + p.optional(p.content(p.until(tool_start))) + tools_parser + p.end();257}258 259common_peg_parser analyze_tools::build_func_parser(common_chat_peg_builder & p, const std::string & name,260                                                    const common_peg_parser & call_id_section, bool have_call_id,261                                                    const common_peg_parser & args,262                                                    std::optional<common_peg_parser> atomic_peek) const {263    auto              open           = p.tool_open(function.name_prefix + p.tool_name(p.literal(name)) + function.name_suffix);264    bool              matched_atomic = false;265    common_peg_parser func_parser    = p.eps();266 267    if (!function.args_separator.empty()) {268        open = open + p.space() + p.literal(function.args_separator);269    }270 271    if (!function.name_suffix.empty()) {272        func_parser    = open + call_id_section + p.space() + args;273        matched_atomic = true;274    } else if (have_call_id) {275        func_parser    = p.atomic(open + call_id_section) + p.space() + args;276        matched_atomic = true;277    } else if (atomic_peek.has_value()) {278        func_parser    = p.atomic(open + call_id_section + p.space() + *atomic_peek) + args;279        matched_atomic = true;280    } else {281        func_parser = open + call_id_section + p.space() + args;282    }283 284    if (!function.close.empty()) {285        func_parser = func_parser + p.space() + p.tool_close(p.literal(function.close));286    } else if (!format.per_call_end.empty()) {287        // When there's no func_close but there is a per_call_end marker, use peek() to ensure288        // we only emit tool_close when we can actually see the closing marker. This prevents289        // premature closing during partial parsing when we've seen e.g. "</" which could be290        // either "</tool_call>" (end) or "<arg_key>" prefix that failed to match.291        // Laguna (v4): the model may emit whitespace between the last </arg_value> and292        // </tool_call> even though the template renders them tight. Tolerate optional293        // leading space in the close lookahead so the tool call still closes.294        auto close_peek = arguments.tolerate_intertag_whitespace295                              ? p.peek(p.space() + p.literal(format.per_call_end))296                              : p.peek(p.literal(format.per_call_end));297        func_parser = func_parser + p.tool_close(close_peek);298    } else {299        func_parser = func_parser + p.tool_close(p.space());  // force this to process tool closing callbacks in mapper300    }301    if (!matched_atomic) {302        func_parser = p.atomic(func_parser);303    }304    return func_parser;305}306 307common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context & ctx) const {308    auto &       p           = ctx.p;309    const auto & inputs      = ctx.inputs;310 311    common_peg_parser tool_choice = p.choice();312 313    foreach_function(inputs.tools, [&](const json & tool) {314        const auto & func   = tool.at("function");315        std::string  name   = func.at("name");316        const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object();317 318        // Build call_id parser based on position (if supported)319        bool have_call_id = false;320        common_peg_parser call_id_section = p.eps();321        if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && !call_id.prefix.empty() &&322            (!call_id.suffix.empty() || !arguments.start.empty())) {323            if (!call_id.suffix.empty()) {324                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix))) + call_id.suffix;325            } else {326                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(arguments.start)));327            }328            have_call_id = true;329        }330        auto args_parser = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema));331        if (!arguments.start.empty()) {332            args_parser = p.literal(arguments.start) + args_parser;333        }334        if (!arguments.end.empty()) {335            args_parser = args_parser + p.literal(arguments.end);336        }337 338        auto atomic_peek = !arguments.start.empty() ? std::optional(p.peek(p.literal(arguments.start))) : std::nullopt;339        auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_parser, atomic_peek);340        tool_choice |= p.rule("tool-" + name, func_parser);341    });342 343    auto require_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;344 345    common_peg_parser tool_calls = p.eps();346 347    if (!format.per_call_start.empty()) {348        auto wrapped_call = format.per_call_start + tool_choice + format.per_call_end;349        if (inputs.parallel_tool_calls) {350            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call));351        } else {352            tool_calls = p.trigger_rule("tool-call", wrapped_call);353        }354        if (!format.section_start.empty()) {355            tool_calls = p.trigger_rule("tool-calls",356                                        p.literal(format.section_start) + p.space() + tool_calls + p.space() +357                                            (format.section_end.empty() ? p.end() : p.literal(format.section_end)));358        }359    } else {360        std::string separator = ", ";  // Default361        if (inputs.parallel_tool_calls) {362            tool_calls = p.trigger_rule("tool-call", format.section_start + tool_choice +363                                                         p.zero_or_more(separator + tool_choice) + format.section_end);364        } else {365            tool_calls = p.trigger_rule("tool-call", format.section_start + tool_choice + format.section_end);366        }367    }368 369    if (!require_calls) {370        tool_calls = p.optional(tool_calls);371    }372 373    std::string trigger_marker       = !format.section_start.empty() ? format.section_start : format.per_call_start;374    auto        content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);375    return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end();376}377 378common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_context & ctx) const {379    auto &       p           = ctx.p;380    const auto & inputs      = ctx.inputs;381 382    auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));383 384    common_peg_parser tool_choice = p.choice();385 386    foreach_function(inputs.tools, [&](const json & tool) {387        const auto &          func       = tool.at("function");388        std::string           name       = func.at("name");389        auto                  params     = func.contains("parameters") ? func.at("parameters") : json::object();390        const auto &          properties = params.contains("properties") ? params.at("properties") : json::object();391 392        std::set<std::string> required;393        if (params.contains("required")) {394            params.at("required").get_to(required);395        }396 397        auto schema_info = common_schema_info();398        schema_info.resolve_refs(params);399 400        // Build parser for each argument, separating required and optional401        std::vector<common_peg_parser> required_parsers;402        std::vector<common_peg_parser> optional_parsers;403        for (const auto & [param_name, param_schema] : properties.items()) {404            bool is_required = required.find(param_name) != required.end();405 406            auto arg =407                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +408                                           arguments.name_suffix) +409                           arguments.value_prefix +410                           (schema_info.resolves_to_string(param_schema) ?411                                p.ac(p.tool_arg_string_value(until_suffix) +412                                    p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :413                                (p.tool_arg_json_value(p.schema(414                                    p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +415                                    p.tool_arg_close(p.literal(arguments.value_suffix)))));416 417            auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);418            if (is_required) {419                required_parsers.push_back(named_arg);420            } else {421                optional_parsers.push_back(named_arg);422            }423        }424 425        // Build required arg sequence in definition order426        common_peg_parser args_seq = p.eps();427        for (size_t i = 0; i < required_parsers.size(); i++) {428            if (i > 0) {429                args_seq = args_seq + p.space();430            }431            args_seq = args_seq + required_parsers[i];432        }433 434        // Build optional args with flexible ordering435        if (!optional_parsers.empty()) {436            common_peg_parser any_opt = p.choice();437            for (const auto & opt : optional_parsers) {438                any_opt |= opt;439            }440            args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);441        }442 443        if (!arguments.start.empty()) {444            args_seq = p.literal(arguments.start) + args_seq;445        }446        if (!arguments.end.empty()) {447            args_seq = args_seq + p.literal(arguments.end);448        }449 450        // Build call_id parser based on position (if supported)451        common_peg_parser call_id_section = p.eps();452        bool have_call_id = false;453        if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && !call_id.prefix.empty() &&454            (!call_id.suffix.empty() || !arguments.start.empty())) {455            have_call_id = true;456            if (!call_id.suffix.empty()) {457                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix)) + call_id.suffix);458            } else {459                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(arguments.start)));460            }461        }462 463        // Only peek for an arg tag when there are required args that must follow.464        // When all args are optional, the model may emit no arg tags at all (#20650).465        auto atomic_peek = (!arguments.name_prefix.empty() && !required_parsers.empty()) ?466            std::optional(p.peek(p.literal(arguments.name_prefix))) : std::nullopt;467        auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_seq, atomic_peek);468        tool_choice |= p.rule("tool-" + name, func_parser);469    });470 471    auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;472 473    common_peg_parser tool_calls = p.eps();474 475    if (!format.per_call_start.empty()) {476        auto wrapped_call = format.per_call_start + p.space() + tool_choice + p.space() + format.per_call_end;477        if (inputs.parallel_tool_calls) {478            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call) + p.space());479        } else {480            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.space());481        }482        if (!format.section_start.empty()) {483            tool_calls = p.trigger_rule("tool-calls",484                                        p.literal(format.section_start) + p.space() + tool_calls + p.space() +485                                            (format.section_end.empty() ? p.end() : p.literal(format.section_end) + p.space()));486        }487    } else {488        std::string separator = ", ";  // Default489 490        if (inputs.parallel_tool_calls) {491            tool_calls = p.trigger_rule("tool-call", format.section_start + p.space() + tool_choice +492                                                         p.zero_or_more(separator + tool_choice) + p.space() +493                                                         format.section_end);494        } else {495            tool_calls = p.trigger_rule(496                "tool-call", format.section_start + p.space() + tool_choice + p.space() + format.section_end);497        }498    }499 500    if (!require_tools) {501        tool_calls = p.optional(tool_calls);502    }503 504    std::string trigger_marker       = !format.section_start.empty() ? format.section_start : format.per_call_start;505    auto        content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);506    return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end();507}508 509}  // namespace autoparser510