echodict/llama.cpp

1#include "chat-auto-parser-helpers.h"2#include "chat-auto-parser.h"3#include "chat-peg-parser.h"4#include "chat.h"5#include "common.h"6#include "json-schema-to-grammar.h"7#include "log.h"8#include "nlohmann/json.hpp"9#include "peg-parser.h"10 11#include <stdexcept>12#include <string>13 14using json = nlohmann::ordered_json;15 16// Helper to iterate over tools/functions17static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {18    for (const auto & tool : tools) {19        if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {20            continue;21        }22        fn(tool);23    }24}25 26namespace autoparser {27 28parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :29    p(p),30    inputs(inputs),31    reasoning_parser(p.eps()) {}32 33common_chat_params peg_generator::generate_parser(const common_chat_template &    tmpl,34                                                  const struct generation_params & inputs) {35    // Run differential analysis to extract template structure36    struct autoparser autoparser;37    autoparser.analyze_template(tmpl);38    return generate_parser(tmpl, inputs, autoparser);39}40 41common_chat_params peg_generator::generate_parser(const common_chat_template &    tmpl,42                                                  const struct generation_params & inputs,43                                                  const autoparser &              autoparser) {44    // Create the result structure45    common_chat_params data;46    data.prompt           = common_chat_template_direct_apply(tmpl, inputs);47    data.format           = COMMON_CHAT_FORMAT_PEG_NATIVE;48    data.preserved_tokens = autoparser.preserved_tokens;49 50    auto parser = autoparser.build_parser(inputs);51    data.parser = parser.save();52 53    // Build grammar if tools are present54    bool has_tools =55        autoparser.tools.format.mode != tool_format::NONE && inputs.tools.is_array() && !inputs.tools.empty();56    std::string trigger_marker = !autoparser.tools.format.section_start.empty() ? autoparser.tools.format.section_start :57                                                                                  autoparser.tools.format.per_call_start;58 59    bool has_response_format = !inputs.json_schema.empty() && inputs.json_schema.is_object();60    bool include_grammar = has_response_format || (has_tools &&61            ((inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO && !trigger_marker.empty()) ||62              inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));63 64    if (include_grammar) {65        data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;66        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {67            foreach_function(inputs.tools, [&](const json & tool) {68                const auto & function = tool.at("function");69                auto         schema   = function.contains("parameters") ? function.at("parameters") : json::object();70                builder.resolve_refs(schema);71            });72            if (has_response_format) {73                auto schema = inputs.json_schema;74                builder.resolve_refs(schema);75            }76            parser.build_grammar(builder, data.grammar_lazy);77        });78 79        // Set grammar triggers based on tool section markers (fall back to per-call markers)80        if (data.grammar_lazy) {81            data.grammar_triggers = {82                { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker }83            };84        }85    }86 87    return data;88}89 90common_peg_arena autoparser::build_parser(const generation_params & inputs) const {91    if (!analysis_complete) {92        throw std::invalid_argument("Cannot call build_parser on autoparser without performing analysis first, call analyze_template(...)");93    }94    return build_chat_peg_parser([&](common_chat_peg_builder & p) {95        parser_build_context ctx(p, inputs);96        bool                 extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;97 98        ctx.extracting_reasoning = extract_reasoning && reasoning.mode != reasoning_mode::NONE;99        ctx.content              = &content;100        ctx.reasoning            = &reasoning;101 102        // Build reasoning parser103        ctx.reasoning_parser = reasoning.build_parser(ctx);104 105        auto parser = p.eps();106 107        bool has_tools           = inputs.tools.is_array() && !inputs.tools.empty();108        bool has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();109        bool pure_content        = reasoning.mode == reasoning_mode::NONE;110 111        if (has_response_format) {112            auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));113            parser = ctx.reasoning_parser + p.space() + p.choice({114                p.literal("```json") + p.space() + response_format + p.space() + p.literal("```"),115                response_format116            }) + p.end();117            pure_content = false;118        } else if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && jinja_caps.supports_tool_calls) {119            parser = tools.build_parser(ctx);120            pure_content = false;121        } else {122            parser = content.build_parser(ctx);123        }124        return pure_content ? p.prefix(inputs.generation_prompt, reasoning.start) + parser : p.prefix(inputs.generation_prompt, reasoning.start) << parser;125    });126}127 128common_peg_parser analyze_reasoning::build_parser(parser_build_context & ctx) const {129    auto & p = ctx.p;130 131    if (!ctx.extracting_reasoning) {132        return p.eps();133    }134 135    if (mode == reasoning_mode::TAG_BASED || mode == reasoning_mode::TOOLS_ONLY) {136        if (!end.empty()) {137            if (!start.empty()) {138                // Standard tag-based: optional(<think>reasoning</think>)139                return p.optional(start + p.reasoning(p.until(end)) + end + p.space());140            }141            // Delimiter-style (empty start)142            return p.optional(p.reasoning(p.until(end)) + end + p.space());143        }144    }145 146    return p.eps();147}148 149common_peg_parser analyze_content::build_parser(parser_build_context & ctx) const {150    auto & p = ctx.p;151 152    if (is_always_wrapped()) {153        if (ctx.extracting_reasoning) {154            return ctx.reasoning_parser + start + p.content(p.until(end)) + end + p.end();155        }156        return p.content(p.until(start)) + start + p.content(p.until(end)) + end + p.end();157    }158    return ctx.reasoning_parser + p.content(p.rest()) + p.end();159}160 161common_peg_parser analyze_content::build_optional_wrapped(parser_build_context & ctx) const {162    auto & p = ctx.p;163 164    if (is_always_wrapped()) {165        return p.optional(start + p.content(p.until(end)) + end);166    }167    return p.eps();168}169 170common_peg_parser analyze_tools::build_parser(parser_build_context & ctx) const {171    switch (format.mode) {172        case tool_format::JSON_NATIVE:173            return build_tool_parser_json_native(ctx);174        case tool_format::TAG_WITH_JSON:175            return build_tool_parser_tag_json(ctx);176        case tool_format::TAG_WITH_TAGGED:177            return build_tool_parser_tag_tagged(ctx);178        default:179            LOG_ERR("[ERROR] Template seems to support tool calls, but failed to determine tool format. Tool calling will not work properly. "180                "Check for a fixed template for your model in the models/templates directory of your llama.cpp installation or "181                "report an issue at https://github.com/ggml-org/llama.cpp/issues\n");182            return ctx.p.eps();183    }184}185 186common_peg_parser analyze_tools::build_tool_parser_json_native(parser_build_context & ctx) const {187    auto &       p           = ctx.p;188    const auto & inputs      = ctx.inputs;189    bool         force_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;190 191    // Build effective field names with dot notation if function_field is set192    std::string name_field = format.name_field;193    std::string args_field = format.args_field;194 195    if (!format.function_field.empty() && format.function_field != "function" &&196        name_field.find('.') == std::string::npos) {197        name_field = format.function_field + "." + name_field;198        args_field = format.function_field + "." + args_field;199    }200 201    auto tools_parser = p.eps();202    if (format.section_start.empty() && !format.per_call_start.empty()) {203        auto single_tool_parser = p.standard_json_tools(204            format.per_call_start, format.per_call_end, inputs.tools, inputs.parallel_tool_calls,205            inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,206            format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order);207        tools_parser = p.trigger_rule("tool-calls", p.one_or_more(single_tool_parser + p.space()));208    } else {209        tools_parser = p.standard_json_tools(210            format.section_start, format.section_end, inputs.tools, inputs.parallel_tool_calls,211            inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped,212            format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order);213    }214 215    // Handle content wrappers if present216    if (ctx.content && ctx.content->is_always_wrapped()) {217        auto wrapped_content = ctx.content->build_optional_wrapped(ctx);218        return ctx.reasoning_parser + wrapped_content + tools_parser + p.end();219    }220 221    std::string tool_start = "{";222    if (!format.section_start.empty()) {223        tool_start = format.section_start;224    } else if (!format.per_call_start.empty()) {225        tool_start = format.per_call_start;226    }227 228    return ctx.reasoning_parser + (force_tools ? p.eps() : p.optional(p.content(p.until(tool_start)))) + tools_parser +229           p.end();230}231 232common_peg_parser analyze_tools::build_func_parser(common_chat_peg_builder & p, const std::string & name,233                                                    const common_peg_parser & call_id_section, bool have_call_id,234                                                    const common_peg_parser & args,235                                                    std::optional<common_peg_parser> atomic_peek) const {236    auto              open           = p.tool_open(function.name_prefix + p.tool_name(p.literal(name)) + function.name_suffix);237    bool              matched_atomic = false;238    common_peg_parser func_parser    = p.eps();239 240    if (!function.name_suffix.empty()) {241        func_parser    = open + call_id_section + p.space() + args;242        matched_atomic = true;243    } else if (have_call_id) {244        func_parser    = p.atomic(open + call_id_section) + p.space() + args;245        matched_atomic = true;246    } else if (atomic_peek.has_value()) {247        func_parser    = p.atomic(open + call_id_section + p.space() + *atomic_peek) + args;248        matched_atomic = true;249    } else {250        func_parser = open + call_id_section + p.space() + args;251    }252 253    if (!function.close.empty()) {254        func_parser = func_parser + p.space() + p.tool_close(p.literal(function.close));255    } else if (!format.per_call_end.empty()) {256        // When there's no func_close but there is a per_call_end marker, use peek() to ensure257        // we only emit tool_close when we can actually see the closing marker. This prevents258        // premature closing during partial parsing when we've seen e.g. "</" which could be259        // either "</tool_call>" (end) or "<arg_key>" prefix that failed to match.260        func_parser = func_parser + p.tool_close(p.peek(p.literal(format.per_call_end)));261    } else {262        func_parser = func_parser + p.tool_close(p.space());  // force this to process tool closing callbacks in mapper263    }264    if (!matched_atomic) {265        func_parser = p.atomic(func_parser);266    }267    return func_parser;268}269 270common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context & ctx) const {271    auto &       p           = ctx.p;272    const auto & inputs      = ctx.inputs;273    bool         force_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;274 275    common_peg_parser tool_choice = p.choice();276 277    foreach_function(inputs.tools, [&](const json & tool) {278        const auto & func   = tool.at("function");279        std::string  name   = func.at("name");280        const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object();281 282        // Build call_id parser based on position (if supported)283        bool have_call_id = false;284        common_peg_parser call_id_section = p.eps();285        if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && !call_id.prefix.empty() &&286            (!call_id.suffix.empty() || !arguments.start.empty())) {287            if (!call_id.suffix.empty()) {288                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix))) + call_id.suffix;289            } else {290                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(arguments.start)));291            }292            have_call_id = true;293        }294        auto args_parser = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema));295        if (!arguments.start.empty()) {296            args_parser = p.literal(arguments.start) + args_parser;297        }298        if (!arguments.end.empty()) {299            args_parser = args_parser + p.literal(arguments.end);300        }301 302        auto atomic_peek = !arguments.start.empty() ? std::optional(p.peek(p.literal(arguments.start))) : std::nullopt;303        auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_parser, atomic_peek);304        tool_choice |= p.rule("tool-" + name, func_parser);305    });306 307    auto require_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;308 309    common_peg_parser tool_calls = p.eps();310 311    if (!format.per_call_start.empty()) {312        auto wrapped_call = format.per_call_start + tool_choice + format.per_call_end;313        if (inputs.parallel_tool_calls) {314            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call));315        } else {316            tool_calls = p.trigger_rule("tool-call", wrapped_call);317        }318        if (!format.section_start.empty()) {319            tool_calls = p.trigger_rule("tool-calls",320                                        p.literal(format.section_start) + p.space() + tool_calls + p.space() +321                                            (format.section_end.empty() ? p.end() : p.literal(format.section_end)));322        }323    } else {324        std::string separator = ", ";  // Default325        if (inputs.parallel_tool_calls) {326            tool_calls = p.trigger_rule("tool-call", format.section_start + tool_choice +327                                                         p.zero_or_more(separator + tool_choice) + format.section_end);328        } else {329            tool_calls = p.trigger_rule("tool-call", format.section_start + tool_choice + format.section_end);330        }331    }332 333    if (!require_calls) {334        tool_calls = p.optional(tool_calls);335    }336 337    std::string trigger_marker       = !format.section_start.empty() ? format.section_start : format.per_call_start;338    auto        content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);339    return ctx.reasoning_parser + (force_tools ? p.eps() : p.optional(p.content(content_before_tools))) + tool_calls +340           p.end();341}342 343common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_context & ctx) const {344    auto &       p           = ctx.p;345    const auto & inputs      = ctx.inputs;346    bool         force_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;347 348    auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));349 350    common_peg_parser tool_choice = p.choice();351 352    foreach_function(inputs.tools, [&](const json & tool) {353        const auto &          func       = tool.at("function");354        std::string           name       = func.at("name");355        auto                  params     = func.contains("parameters") ? func.at("parameters") : json::object();356        const auto &          properties = params.contains("properties") ? params.at("properties") : json::object();357 358        std::set<std::string> required;359        if (params.contains("required")) {360            params.at("required").get_to(required);361        }362 363        auto schema_info = common_schema_info();364        schema_info.resolve_refs(params);365 366        // Build parser for each argument, separating required and optional367        std::vector<common_peg_parser> required_parsers;368        std::vector<common_peg_parser> optional_parsers;369        for (const auto & [param_name, param_schema] : properties.items()) {370            bool is_required = required.find(param_name) != required.end();371 372            auto arg =373                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +374                                           arguments.name_suffix) +375                           arguments.value_prefix +376                           (schema_info.resolves_to_string(param_schema) ?377                                p.tool_arg_string_value(p.schema(until_suffix,378                                                                 "tool-" + name + "-arg-" + param_name + "-schema",379                                                                 param_schema, true)) :380                                p.tool_arg_json_value(p.schema(381                                    p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +382                                    p.space()) +383                           p.tool_arg_close(p.literal(arguments.value_suffix)));384 385            auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);386            if (is_required) {387                required_parsers.push_back(named_arg);388            } else {389                optional_parsers.push_back(named_arg);390            }391        }392 393        // Build required arg sequence in definition order394        common_peg_parser args_seq = p.eps();395        for (size_t i = 0; i < required_parsers.size(); i++) {396            if (i > 0) {397                args_seq = args_seq + p.space();398            }399            args_seq = args_seq + required_parsers[i];400        }401 402        // Build optional args with flexible ordering403        if (!optional_parsers.empty()) {404            common_peg_parser any_opt = p.choice();405            for (const auto & opt : optional_parsers) {406                any_opt |= opt;407            }408            args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);409        }410 411        if (!arguments.start.empty()) {412            args_seq = p.literal(arguments.start) + args_seq;413        }414        if (!arguments.end.empty()) {415            args_seq = args_seq + p.literal(arguments.end);416        }417 418        // Build call_id parser based on position (if supported)419        common_peg_parser call_id_section = p.eps();420        bool have_call_id = false;421        if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && !call_id.prefix.empty() &&422            (!call_id.suffix.empty() || !arguments.start.empty())) {423            have_call_id = true;424            if (!call_id.suffix.empty()) {425                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix)) + call_id.suffix);426            } else {427                call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(arguments.start)));428            }429        }430 431        // Only peek for an arg tag when there are required args that must follow.432        // When all args are optional, the model may emit no arg tags at all (#20650).433        auto atomic_peek = (!arguments.name_prefix.empty() && !required_parsers.empty()) ?434            std::optional(p.peek(p.literal(arguments.name_prefix))) : std::nullopt;435        auto func_parser = build_func_parser(p, name, call_id_section, have_call_id, args_seq, atomic_peek);436        tool_choice |= p.rule("tool-" + name, func_parser);437    });438 439    auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;440 441    common_peg_parser tool_calls = p.eps();442 443    if (!format.per_call_start.empty()) {444        auto wrapped_call = format.per_call_start + p.space() + tool_choice + p.space() + format.per_call_end;445        if (inputs.parallel_tool_calls) {446            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call) + p.space());447        } else {448            tool_calls = p.trigger_rule("tool-call", wrapped_call + p.space());449        }450        if (!format.section_start.empty()) {451            tool_calls = p.trigger_rule("tool-calls",452                                        p.literal(format.section_start) + p.space() + tool_calls + p.space() +453                                            (format.section_end.empty() ? p.end() : p.literal(format.section_end) + p.space()));454        }455    } else {456        std::string separator = ", ";  // Default457 458        if (inputs.parallel_tool_calls) {459            tool_calls = p.trigger_rule("tool-call", format.section_start + p.space() + tool_choice +460                                                         p.zero_or_more(separator + tool_choice) + p.space() +461                                                         format.section_end);462        } else {463            tool_calls = p.trigger_rule(464                "tool-call", format.section_start + p.space() + tool_choice + p.space() + format.section_end);465        }466    }467 468    if (!require_tools) {469        tool_calls = p.optional(tool_calls);470    }471 472    std::string trigger_marker       = !format.section_start.empty() ? format.section_start : format.per_call_start;473    auto        content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker);474    return ctx.reasoning_parser + (force_tools ? p.eps() : p.optional(p.content(content_before_tools))) + tool_calls +475           p.end();476}477 478}  // namespace autoparser479