Brunobkr/llama.cpp_AlgMor24_github

1#include "chat-peg-parser.h"2 3#include "chat-auto-parser.h"4#include "ggml.h"5#include "peg-parser.h"6 7#include <nlohmann/json.hpp>8 9#include <cstdint>10#include <functional>11 12using ordered_json = nlohmann::ordered_json;13 14static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {15    int count = 0;16    while (!sv.empty() && std::isspace(static_cast<unsigned char>(sv.back()))) {17        if (max != -1 && count >= max) {18            break;19        }20        sv.remove_suffix(1);21        count++;22    }23    return sv;24}25 26static std::string_view trim_leading_space(std::string_view sv, int max = -1) {27    int count = 0;28    while (!sv.empty() && std::isspace(static_cast<unsigned char>(sv.front()))) {29        if (max != -1 && count >= max) {30            break;31        }32        sv.remove_prefix(1);33        count++;34    }35    return sv;36}37 38static std::string_view trim(std::string_view sv) {39    return trim_trailing_space(trim_leading_space(sv, 1));40}41 42// Count the number of unclosed '{' braces in a JSON-like string,43// properly skipping braces inside quoted strings.44static int json_brace_depth(const std::string & s) {45    int  depth     = 0;46    bool in_string = false;47    bool escaped   = false;48    for (char c : s) {49        if (escaped) {50            escaped = false;51            continue;52        }53        if (c == '\\' && in_string) {54            escaped = true;55            continue;56        }57        if (c == '"') {58            in_string = !in_string;59            continue;60        }61        if (!in_string) {62            if (c == '{') {63                depth++;64            } else if (c == '}') {65                depth--;66            }67        }68    }69    return depth;70}71 72// JSON-escape a string and return the inner content (without surrounding quotes).73static std::string escape_json_string_inner(const std::string & s) {74    std::string escaped = ordered_json(s).dump();75    if (escaped.size() >= 2 && escaped.front() == '"' && escaped.back() == '"') {76        return escaped.substr(1, escaped.size() - 2);77    }78    return escaped;79}80 81// Convert Python-style single-quoted strings to JSON double-quoted strings82// Only converts outer string delimiters, properly handling escape sequences:83// - {'key': 'value'} -> {"key": "value"}84// - {'code': 'print(\'hello\')'} -> {"code": "print('hello')"}85// - {'msg': 'He said "hi"'} -> {"msg": "He said \"hi\""}86static std::string normalize_quotes_to_json(const std::string & input) {87    std::string result;88    result.reserve(input.size() + 16);  // May need extra space for escaping89 90    bool in_single_quoted = false;91    bool in_double_quoted = false;92 93    auto is_word_char = [](char ch) { return std::isalnum(static_cast<unsigned char>(ch)) || ch == '_'; };94 95    for (size_t i = 0; i < input.size(); ++i) {96        char c = input[i];97 98        // Handle escape sequences99        if (c == '\\' && i + 1 < input.size()) {100            char next = input[i + 1];101 102            if (in_single_quoted) {103                // Inside a single-quoted string being converted to double quotes104                if (next == '\'') {105                    // \' -> ' (escaped single quote becomes unescaped in double-quoted string)106                    result += '\'';107                    ++i;108                    continue;109                }110                if (next == '"') {111                    // \" stays as \" (already escaped, works in double-quoted string)112                    result += "\\\"";113                    ++i;114                    continue;115                }116                // Other escapes (\n, \\, etc.): pass through both characters117                result += c;118                result += next;119                ++i;120                continue;121            }122 123            if (in_double_quoted) {124                // Inside a double-quoted string - pass through escape sequences as-is125                result += c;126                result += next;127                ++i;128                continue;129            }130 131            // Outside any string - just pass through the backslash132            result += c;133            continue;134        }135 136        // Handle quote characters137        if (c == '"') {138            if (in_single_quoted) {139                // Unescaped double quote inside single-quoted string -> must escape for JSON140                result += "\\\"";141            } else {142                // Double quote as string delimiter or outside strings143                in_double_quoted = !in_double_quoted;144                result += c;145            }146        } else if (c == '\'') {147            if (in_double_quoted) {148                // Single quote inside double-quoted string -> pass through149                result += c;150            } else if (in_single_quoted) {151                // Closing single quote -> convert to double quote152                in_single_quoted = false;153                result += '"';154            } else {155                // Opening single quote -> convert to double quote156                in_single_quoted = true;157                result += '"';158            }159        } else if (!in_single_quoted && !in_double_quoted && (c == 'T' || c == 'F' || c == 'N') &&160                   (i == 0 || !is_word_char(input[i - 1]))) {161            // Python literals -> JSON; prefix match keeps streamed partials monotonic.162            static constexpr std::pair<std::string_view, std::string_view> literals[] = {163                { "True", "true" }, { "False", "false" }, { "None", "null" },164            };165            size_t n = 0;166            while (i + n < input.size() && is_word_char(input[i + n])) {167                ++n;168            }169            std::string_view token(input.data() + i, n);170            bool matched = false;171            for (const auto & [py, js] : literals) {172                if (py.substr(0, n) == token) {173                    result += js.substr(0, n);174                    i += n - 1;175                    matched = true;176                    break;177                }178            }179            if (!matched) {180                result += c;181            }182        } else {183            result += c;184        }185    }186 187    return result;188}189 190void tag_based_peg_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) {191    arena.visit(result, [this](const common_peg_ast_node & node) {192        if (!node.tag.empty()) {193            tags[node.tag] = std::string(node.text);194        }195    });196}197 198tagged_parse_result tagged_peg_parser::parse_and_extract(const std::string & input, common_peg_parse_flags extra_flags) const {199    common_peg_parse_context ctx(input, flags | extra_flags);200    auto parse_result = arena.parse(ctx);201 202    tag_based_peg_mapper mapper;203    mapper.from_ast(ctx.ast, parse_result);204 205    return { std::move(parse_result), std::move(mapper.tags) };206}207 208tagged_parse_result tagged_peg_parser::parse_anywhere_and_extract(const std::string & input) const {209    if (input.empty()) {210        return parse_and_extract(input);211    }212    for (size_t i = 0; i < input.size(); i++) {213        common_peg_parse_context ctx(input, flags);214        auto parse_result = arena.parse(ctx, i);215        if (parse_result.success() || i == input.size() - 1) {216            tag_based_peg_mapper mapper;217            mapper.from_ast(ctx.ast, parse_result);218            return { std::move(parse_result), std::move(mapper.tags) };219        }220    }221    GGML_ABORT("Should not happen");222}223 224tagged_peg_parser build_tagged_peg_parser(225    const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {226    common_peg_parser_builder builder;227    builder.set_root(fn(builder));228    return { builder.build() };229}230 231common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::string &       tag_name,232                                                                 const std::string &       marker,233                                                                 const common_peg_parser & p) {234    if (marker.empty()) {235        return zero_or_more(choice({ p, rule(tag_name, content(any())) }));236    }237    auto content_chunk = rule(tag_name, content(negate(literal(marker)) + any() + until(marker)));238    return zero_or_more(choice({ p, content_chunk }));239}240 241common_peg_parser common_chat_peg_builder::permute(const std::string &                    rule_prefix,242                                                   const std::vector<common_peg_parser> & parsers) {243    if (parsers.empty()) {244        return eps();245    }246 247    if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {248        return sequence(parsers);249    }250 251    std::map<uint32_t, common_peg_parser>      rules;252    std::function<common_peg_parser(uint32_t)> remaining_of;253 254    remaining_of = [&](uint32_t remaining) -> common_peg_parser {255        if (remaining == 0) {256            return eps();257        }258 259        auto cached = rules.find(remaining);260        if (cached != rules.end()) {261            return cached->second;262        }263 264        auto alternatives = choice();265        for (size_t i = 0; i < parsers.size(); i++) {266            const uint32_t bit = 1u << i;267            if (remaining & bit) {268                alternatives |= parsers[i] + remaining_of(remaining & ~bit);269            }270        }271 272        return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;273    };274 275    return remaining_of((1u << parsers.size()) - 1);276}277 278std::string & common_chat_peg_mapper::args_target() {279    return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;280}281 282std::string common_chat_peg_mapper::normalize_container_value(const std::string & input) {283    return normalize_quotes_to_json(input);284}285 286void common_chat_peg_mapper::from_ast(const common_peg_ast_arena &    arena,287                                      const common_peg_parse_result & parse_result_arg) {288    arena.visit(parse_result_arg, [this](const common_peg_ast_node & node) { map(node); });289    // Flush any pending tool call that was started but never got a name290    // This happens during partial parsing when the tool call is incomplete291    if (pending_tool_call.has_value() && !pending_tool_call->name.empty()) {292        if (!args_buffer.empty()) {293            pending_tool_call->arguments = args_buffer;294        }295        if (closing_quote_pending && !pending_tool_call->arguments.empty()) {296            pending_tool_call->arguments += "\"";297        }298        result.tool_calls.push_back(pending_tool_call.value());299        pending_tool_call.reset();300    }301 302    // Discard whitespace-only reasoning content (e.g. from <think></think> prefill)303    if (!result.reasoning_content.empty()) {304        bool all_whitespace = true;305        for (char c : result.reasoning_content) {306            if (c != ' ' && c != '\n' && c != '\r' && c != '\t') {307                all_whitespace = false;308                break;309            }310        }311        if (all_whitespace) {312            result.reasoning_content.clear();313        }314    }315}316 317void common_chat_peg_mapper::map(const common_peg_ast_node & node) {318    // Handle reasoning/content tags319    bool is_reasoning = node.tag == common_chat_peg_builder::REASONING;320    bool is_content   = node.tag == common_chat_peg_builder::CONTENT;321 322    if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here323        result.reasoning_content += std::string(node.text);324    }325 326    if (is_content) {327        // Concatenate content from multiple content nodes (e.g., when reasoning markers328        // are preserved before content markers in reasoning_format=NONE mode)329        result.content += std::string(node.text);330    }331 332    // Handle tool-related tags (supporting both JSON and tagged formats)333    bool is_tool_open  = node.tag == common_chat_peg_builder::TOOL_OPEN;334    bool is_tool_close = node.tag == common_chat_peg_builder::TOOL_CLOSE;335    bool is_tool_name  = node.tag == common_chat_peg_builder::TOOL_NAME;336    bool is_tool_id    = node.tag == common_chat_peg_builder::TOOL_ID;337    bool is_tool_args  = node.tag == common_chat_peg_builder::TOOL_ARGS;338    bool is_arg_open   = node.tag == common_chat_peg_builder::TOOL_ARG_OPEN;339    bool is_arg_close  = node.tag == common_chat_peg_builder::TOOL_ARG_CLOSE;340    bool is_arg_name         = node.tag == common_chat_peg_builder::TOOL_ARG_NAME;341    bool is_arg_value        = node.tag == common_chat_peg_builder::TOOL_ARG_VALUE;342    bool is_arg_string_value = node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE;343 344    if (is_tool_open) {345        pending_tool_call     = common_chat_tool_call();346        current_tool          = &pending_tool_call.value();347        arg_count             = 0;348        args_buffer.clear();349        closing_quote_pending = false;350    }351 352    if (is_tool_id && current_tool) {353        auto text = trim_trailing_space(node.text);354        if (text.size() >= 2 && text.front() == '"' && text.back() == '"') {355            text = text.substr(1, text.size() - 2);356        }357        current_tool->id = std::string(text);358    }359 360    if (is_tool_name && current_tool) {361        current_tool->name = std::string(trim_trailing_space(node.text));362        // Now that we have the name, populate the arguments from the buffer363        if (!args_buffer.empty()) {364            current_tool->arguments = args_buffer;365            args_buffer.clear();366        } else if (current_tool->arguments.empty()) {367            current_tool->arguments = "{";368        }369        // Add the tool call to results so streaming can see it370        if (pending_tool_call.has_value()) {371            result.tool_calls.push_back(pending_tool_call.value());372            pending_tool_call.reset();373            current_tool = &result.tool_calls.back();374        }375    }376 377    if (is_tool_args && current_tool) {378        // For JSON format: arguments come as a complete JSON object379        // For tagged format: built up from individual arg_name/arg_value nodes380        auto text = trim_trailing_space(node.text);381        if (!text.empty() && text.front() == '{') {382            args_target() = std::string(text);383        }384    }385 386    if (is_arg_open) {387        closing_quote_pending = false;388    }389 390    if (is_arg_name && current_tool) {391        std::string arg_entry;392        if (arg_count > 0) {393            arg_entry = ",";394        }395        arg_entry += ordered_json(trim(node.text)).dump() + ":";396        ++arg_count;397 398        auto & target = args_target();399        if (target.empty()) {400            target = "{";401        }402        target += arg_entry;403    }404 405    if ((is_arg_value || is_arg_string_value) && current_tool) {406        std::string value_content = std::string(node.text);407 408        std::string value_to_add;409        if (value_content.empty() && is_arg_string_value) {410            // Empty string value - arg_close will add the closing quote411            value_to_add          = "\"";412            closing_quote_pending = true;413        } else if (!value_content.empty() && is_arg_string_value) {414            // Schema declares this as string type - always treat as literal string value415            if (!closing_quote_pending) {416                value_to_add          = "\"";417                closing_quote_pending = true;418            }419            value_to_add += escape_json_string_inner(value_content);420        } else if (!value_content.empty()) {421            // Pythonic scalars/containers -> JSON.422            value_to_add += normalize_container_value(value_content);423        }424 425        args_target() += value_to_add;426    }427 428    if (is_arg_close && current_tool) {429        if (closing_quote_pending) {430            args_target() += "\"";431            closing_quote_pending = false;432        }433    }434 435    if (is_tool_close && current_tool) {436        // Flush buffer to arguments if tool name was never seen437        if (current_tool->name.empty() && !args_buffer.empty()) {438            current_tool->arguments = args_buffer;439            args_buffer.clear();440        }441        // Close any pending string quote442        if (closing_quote_pending) {443            current_tool->arguments += "\"";444            closing_quote_pending = false;445        }446        // Close any unclosed braces (accounts for nested objects)447        for (int d = json_brace_depth(current_tool->arguments); d > 0; d--) {448            current_tool->arguments += "}";449        }450        // Add tool call to results if named; otherwise discard451        if (pending_tool_call.has_value()) {452            if (!current_tool->name.empty()) {453                result.tool_calls.push_back(pending_tool_call.value());454            }455            pending_tool_call.reset();456        }457    }458}459 460common_peg_parser common_chat_peg_builder::standard_constructed_tools(461    const std::map<std::string, std::string> & markers,462    const ordered_json &                       tools,463    bool                                       parallel_tool_calls,464    bool                                       force_tool_calls) {465    if (!tools.is_array() || tools.empty()) {466        return eps();467    }468 469    // Extract markers with defaults470    auto get_marker = [&markers](const std::string & key, const std::string & default_val = "") -> std::string {471        auto it = markers.find(key);472        return it != markers.end() ? it->second : default_val;473    };474 475    std::string section_start    = get_marker("tool_call_start_marker", "<tool_call>");476    std::string section_end      = get_marker("tool_call_end_marker", "</tool_call>");477    std::string func_opener      = get_marker("function_opener", "<function=");478    std::string func_name_suffix = get_marker("function_name_suffix", ">");479    std::string func_closer      = get_marker("function_closer", "</function>");480    std::string param_key_prefix = get_marker("parameter_key_prefix", "<param=");481    std::string param_key_suffix = get_marker("parameter_key_suffix", ">");482    std::string param_closer     = get_marker("parameter_closer", "</param>");483 484    // Build tool choices for tagged format485    auto tool_choices = choice();486 487    for (const auto & tool_def : tools) {488        if (!tool_def.contains("function")) {489            continue;490        }491        const auto &   function = tool_def.at("function");492        std::string    name     = function.at("name");493        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();494 495        // Build argument parsers496        auto args = eps();497        if (params.contains("properties") && !params["properties"].empty()) {498            auto arg_choice = choice();499            for (const auto & el : params["properties"].items()) {500                const std::string & prop_name = el.key();501 502                auto arg_name_parser =503                    choice({ literal(prop_name), literal("\"" + prop_name + "\""), literal("'" + prop_name + "'") });504 505                auto arg_rule = tool_arg(tool_arg_open(literal(param_key_prefix)) + tool_arg_name(arg_name_parser) +506                                         literal(param_key_suffix) + tool_arg_value(until(param_closer)) +507                                         tool_arg_close(literal(param_closer)));508                arg_choice |= arg_rule;509            }510            args = zero_or_more(arg_choice + space());511        }512 513        // Build function parser: <function=name>args</function>514        auto tool_parser = tool(tool_open(literal(func_opener) + tool_name(literal(name)) + literal(func_name_suffix)) +515                                space() + tool_args(args) + space() + tool_close(literal(func_closer)));516 517        tool_choices |= rule("tool-" + name, tool_parser);518    }519 520    // Build the section with markers521    auto section =522        parallel_tool_calls ?523            trigger_rule("tool-call", literal(section_start) + space() + one_or_more(tool_choices + space()) +524                                          literal(section_end)) :525            trigger_rule("tool-call", literal(section_start) + space() + tool_choices + space() + literal(section_end));526 527    return force_tool_calls ? section : optional(section);528}529 530// Like python_value(), but the leaf also accepts JSON-cased true/false/null, used by LFM2/LFM2.5531common_peg_parser common_chat_peg_builder::python_or_json_value() {532    return rule("python-or-json-value", [this]() {533        auto ws    = space();534        auto value = python_or_json_value();535 536        auto member  = sequence({ python_string(), ws, literal(":"), ws, value });537        auto members = sequence({ member, zero_or_more(sequence({ ws, literal(","), ws, member })) });538        auto dict    = rule("python-or-json-dict", [&]() {539            return sequence({ literal("{"), ws, choice({ literal("}"), sequence({ members, ws, literal("}") }) }), ws });540        });541 542        auto elements = sequence({ value, zero_or_more(sequence({ literal(","), ws, value })) });543        auto array    = rule("python-or-json-array", [&]() {544            return sequence({ literal("["), ws, choice({ literal("]"), sequence({ elements, ws, literal("]") }) }), ws });545        });546 547        return choice({ dict, array, python_string(), python_number(),548                        python_bool(), python_null(), json_bool(), json_null() });549    });550}551 552// Python-style tool calls: name(arg1="value1", arg2=123)553// Used only by LFM2 for now, so we don't merge it into autoparser554common_peg_parser common_chat_peg_builder::python_style_tool_calls(555    const ordered_json & tools,556    bool                 parallel_tool_calls,557    bool                 allow_json_literals) {558    if (!tools.is_array() || tools.empty()) {559        return eps();560    }561 562    auto tool_choices = choice();563 564    for (const auto & tool_def : tools) {565        if (!tool_def.contains("function")) {566            continue;567        }568        const auto &   function = tool_def.at("function");569        std::string    name     = function.at("name");570        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();571 572        auto args = eps();573        if (params.contains("properties") && !params["properties"].empty()) {574            auto arg_choice = choice();575            for (const auto & el : params["properties"].items()) {576                const std::string & prop_name = el.key();577                const auto & prop_def = el.value();578                bool is_string_type = (prop_def.contains("type") && prop_def["type"] == "string");579 580                auto arg_name_parser = literal(prop_name);581 582                common_peg_parser arg_value_parser = eps();583                // Quoted literal as a value: normalize_quotes_to_json preserves escapes.584                auto string_value_parser = tool_arg_value(choice({585                    literal("\"") + string_content('"') + literal("\""),586                    literal("'") + string_content('\'') + literal("'")587                }));588 589                if (is_string_type) {590                    arg_value_parser = string_value_parser;591                } else {592                    arg_value_parser = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());593                }594 595                // Full argument: name="value" or name=value596                auto arg_rule = tool_arg(597                    tool_arg_open(eps()) +598                    tool_arg_name(arg_name_parser) +599                    literal("=") +600                    arg_value_parser +601                    tool_arg_close(eps())602                );603                arg_choice |= arg_rule;604            }605 606            args = arg_choice + zero_or_more("," + space() + arg_choice);607        }608 609        auto tool_parser = tool(tool_open(tool_name(literal(name)) + literal("(")) +610            space() + tool_args(args) + space() + tool_close(literal(")"))611        );612 613        tool_choices |= rule("tool-" + name, tool_parser);614    }615 616    if (parallel_tool_calls) {617        return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]";618    }619    return "[" + space() + tool_choices + space() + "]";620}621 622// Helper: Parse dot notation key into prefix and field name623static std::pair<std::string, std::string> parse_key_spec(const std::string & key) {624    auto dot_pos = key.find('.');625    if (dot_pos == std::string::npos) {626        return {"", key};  // Top-level field627    }628    return {key.substr(0, dot_pos), key.substr(dot_pos + 1)};629}630 631// Mode 1: function_is_key — parse {"function_name": {...}}632common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(633    const ordered_json & tools,634    const std::string &  args_key,635    const std::string &  effective_args_key,636    const std::string &  call_id_key,637    const std::string &  gen_call_id_key) {638 639    auto tool_choices = choice();640 641    for (const auto & tool_def : tools) {642        if (!tool_def.contains("function")) {643            continue;644        }645        const auto &   function = tool_def.at("function");646        std::string    name     = function.at("name");647        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();648 649        // Build inner object fields650        std::vector<common_peg_parser> inner_fields;651 652        if (!call_id_key.empty()) {653            auto id_parser = atomic(654                literal("\"" + call_id_key + "\"") + space() + literal(":") + space() +655                literal("\"") + tool_id(string_content('"')) + literal("\"")656            );657            inner_fields.push_back(optional(id_parser + space() + optional(literal(",") + space())));658        }659 660        if (!gen_call_id_key.empty()) {661            auto gen_id_parser = atomic(662                literal("\"" + gen_call_id_key + "\"") + space() + literal(":") + space() +663                choice({664                    literal("\"") + tool_id(string_content('"')) + literal("\""),665                    tool_id(json_number())666                })667            );668            inner_fields.push_back(optional(gen_id_parser + space() + optional(literal(",") + space())));669        }670 671        // Arguments — either wrapped in args_key or parsed directly672        common_peg_parser args_parser = eps();673        if (args_key.empty()) {674            args_parser = tool_args(schema(json(), "tool-" + name + "-schema", params));675        } else {676            args_parser = literal("\"" + effective_args_key + "\"") + space() + literal(":") + space() +677                          tool_args(schema(json(), "tool-" + name + "-schema", params));678        }679        inner_fields.push_back(args_parser);680 681        // Build inner object parser682        common_peg_parser inner_object = eps();683        if (args_key.empty() && inner_fields.size() == 1) {684            inner_object = inner_fields[0];685        } else {686            inner_object = literal("{") + space();687            for (size_t i = 0; i < inner_fields.size(); i++) {688                inner_object = inner_object + inner_fields[i];689                if (i < inner_fields.size() - 1) {690                    inner_object = inner_object + space();691                }692            }693            inner_object = inner_object + space() + literal("}");694        }695 696        auto tool_parser = tool(697            tool_open(literal("{")) + space() +698            literal("\"") + tool_name(literal(name)) + literal("\"") +699            space() + literal(":") + space() +700            inner_object +701            space() + tool_close(literal("}"))702        );703 704        tool_choices |= rule("tool-" + name, tool_parser);705    }706 707    return tool_choices;708}709 710// Mode 2: Nested keys (dot notation like "function.name")711common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(712    const ordered_json & tools,713    const std::string &  effective_name_key,714    const std::string &  effective_args_key,715    const std::string &  call_id_key,716    const std::string &  gen_call_id_key) {717 718    auto tool_choices = choice();719 720    auto name_spec = parse_key_spec(effective_name_key);721    auto args_spec = parse_key_spec(effective_args_key);722 723    std::string nested_prefix     = !name_spec.first.empty() ? name_spec.first  : args_spec.first;724    std::string nested_name_field = !name_spec.first.empty() ? name_spec.second  : effective_name_key;725    std::string nested_args_field = !args_spec.first.empty() ? args_spec.second  : effective_args_key;726 727    for (const auto & tool_def : tools) {728        if (!tool_def.contains("function")) {729            continue;730        }731        const auto &   function = tool_def.at("function");732        std::string    name     = function.at("name");733        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();734 735        auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +736                          atomic(literal("\"") + tool_name(literal(name)) + literal("\""));737        auto nested_args = literal("\"" + nested_args_field + "\"") + space() + literal(":") + space() +738                          tool_args(schema(json(), "tool-" + name + "-schema", params));739 740        auto nested_object = literal("{") + space() +741                            nested_name + space() + literal(",") + space() +742                            nested_args +743                            space() + literal("}");744 745        // Format: { id?, "function": {...} }746        auto tool_parser_body = tool_open(literal("{")) + space();747 748        if (!call_id_key.empty()) {749            auto id_spec = parse_key_spec(call_id_key);750            if (id_spec.first.empty()) {751                auto id_parser = atomic(752                    literal("\"" + call_id_key + "\"") + space() + literal(":") + space() +753                    literal("\"") + tool_id(string_content('"')) + literal("\"")754                );755                tool_parser_body = tool_parser_body + optional(id_parser + space() + literal(",") + space());756            }757        }758 759        if (!gen_call_id_key.empty()) {760            auto gen_id_spec = parse_key_spec(gen_call_id_key);761            if (gen_id_spec.first.empty()) {762                auto gen_id_parser = atomic(763                    literal("\"" + gen_call_id_key + "\"") + space() + literal(":") + space() +764                    choice({765                        literal("\"") + tool_id(string_content('"')) + literal("\""),766                        tool_id(json_number())767                    })768                );769                tool_parser_body = tool_parser_body + optional(gen_id_parser + space() + literal(",") + space());770            }771        }772 773        auto nested_field = literal("\"" + nested_prefix + "\"") + space() + literal(":") + space() + nested_object;774        tool_parser_body = tool_parser_body + nested_field + space() + tool_close(literal("}"));775 776        tool_choices |= rule("tool-" + name, tool(tool_parser_body));777    }778 779    return tool_choices;780}781 782// Mode 3: Flat keys with optional ID fields and parameter ordering783common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(784    const ordered_json &             tools,785    const std::string &              effective_name_key,786    const std::string &              effective_args_key,787    const std::string &              call_id_key,788    const std::string &              gen_call_id_key,789    const std::vector<std::string> & parameters_order,790    bool                             accept_openai_wrapper) {791 792    auto tool_choices    = choice();793    auto name_key_parser = literal("\"" + effective_name_key + "\"");794    auto args_key_parser = literal("\"" + effective_args_key + "\"");795 796    for (const auto & tool_def : tools) {797        if (!tool_def.contains("function")) {798            continue;799        }800        const auto &   function = tool_def.at("function");801        std::string    name     = function.at("name");802        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();803 804        auto tool_name_ = name_key_parser + space() + literal(":") + space() +805                         atomic(literal("\"") + tool_name(literal(name)) + literal("\""));806        auto tool_args_ = args_key_parser + space() + literal(":") + space() +807                         tool_args(schema(json(), "tool-" + name + "-schema", params));808 809        // Build ID parsers if keys are provided810        common_peg_parser id_parser = eps();811        if (!call_id_key.empty()) {812            id_parser = atomic(813                literal("\"" + call_id_key + "\"") + space() + literal(":") + space() +814                choice({815                    literal("\"") + tool_id(string_content('"')) + literal("\""),816                    tool_id(json_number())817                })818            );819        }820 821        common_peg_parser gen_id_parser = eps();822        if (!gen_call_id_key.empty()) {823            gen_id_parser = atomic(824                literal("\"" + gen_call_id_key + "\"") + space() + literal(":") + space() +825                choice({826                    literal("\"") + tool_id(string_content('"')) + literal("\""),827                    tool_id(json_number())828                })829            );830        }831 832        // Create (parser, key) pairs for all fields, then sort by parameters_order833        std::vector<std::pair<common_peg_parser, std::string>> parser_pairs;834        parser_pairs.emplace_back(tool_name_, effective_name_key);835        parser_pairs.emplace_back(tool_args_, effective_args_key);836        if (!call_id_key.empty()) {837            parser_pairs.emplace_back(optional(id_parser), call_id_key);838        }839        if (!gen_call_id_key.empty()) {840            parser_pairs.emplace_back(optional(gen_id_parser), gen_call_id_key);841        }842 843        std::sort(parser_pairs.begin(), parser_pairs.end(),844            [&parameters_order](const auto & a, const auto & b) {845                auto pos_a = std::find(parameters_order.begin(), parameters_order.end(), a.second);846                auto pos_b = std::find(parameters_order.begin(), parameters_order.end(), b.second);847                size_t idx_a = (pos_a == parameters_order.end()) ? parameters_order.size() : std::distance(parameters_order.begin(), pos_a);848                size_t idx_b = (pos_b == parameters_order.end()) ? parameters_order.size() : std::distance(parameters_order.begin(), pos_b);849                return idx_a < idx_b;850            });851 852        // accept an optional leading "type": "function" field when the model emits the OpenAI wrapper853        common_peg_parser type_field = eps();854        if (accept_openai_wrapper) {855            type_field = optional(literal("\"type\"") + space() + literal(":") + space() +856                                  literal("\"function\"") + space() + literal(",") + space());857        }858        auto ordered_body = tool_open(literal("{")) + space() + type_field;859        for (size_t i = 0; i < parser_pairs.size(); i++) {860            ordered_body = ordered_body + parser_pairs[i].first;861            if (i < parser_pairs.size() - 1) {862                ordered_body = ordered_body + space() + literal(",") + space();863            }864        }865        ordered_body = ordered_body + space() + tool_close(literal("}"));866 867        tool_choices |= rule("tool-" + name, tool(ordered_body));868    }869 870    return tool_choices;871}872 873common_peg_parser common_chat_peg_builder::prefix(const std::string & s, const std::string & delimiter) {874    if (s.empty()) {875        return eps();876    }877    if (delimiter.empty()) {878        return literal(s);879    }880    return literal(s.substr(0, s.find(delimiter)));881}882 883common_peg_parser common_chat_peg_builder::optspace(const std::string & tag) {884    auto parser = eps();885    size_t end_of_prefix_space = tag.size();886    size_t start_of_suffix_space = tag.size();887    for (size_t i = 0; i < tag.size(); i++) {888        if (!std::isspace(tag[i])) {889            end_of_prefix_space = i;890            break;891        }892    }893    for (size_t i = tag.size(); i > 0; i--) {894        if (!std::isspace(tag[i - 1])) {895            start_of_suffix_space = i;896            break;897        }898    }899    for (size_t i = 0; i < end_of_prefix_space; i++) {900        parser += optional(literal(std::string(1, tag[i])));901    }902    parser += literal(tag.substr(end_of_prefix_space, start_of_suffix_space - end_of_prefix_space));903    for (size_t i = start_of_suffix_space; i < tag.size(); i++) {904        parser += optional(literal(std::string(1, tag[i])));905    }906    return parser;907}908 909common_peg_parser common_chat_peg_builder::standard_json_tools(910                                                       const std::string &              section_start,911                                                       const std::string &              section_end,912                                                       const ordered_json &             tools,913                                                       bool                             parallel_tool_calls,914                                                       bool                             force_tool_calls,915                                                       const std::string &              name_key,916                                                       const std::string &              args_key,917                                                       bool                             array_wrapped,918                                                       bool                             function_is_key,919                                                       const std::string &              call_id_key,920                                                       const std::string &              gen_call_id_key,921                                                       const std::vector<std::string> & parameters_order,922                                                       bool                             accept_openai_wrapper) {923    if (!tools.is_array() || tools.empty()) {924        return eps();925    }926 927    std::string effective_name_key = name_key.empty() ? "name" : name_key;928    std::string effective_args_key = args_key.empty() ? "arguments" : args_key;929 930    // Dispatch to the appropriate builder based on the JSON layout mode931    common_peg_parser tool_choices = eps();932    if (function_is_key) {933        tool_choices = build_json_tools_function_is_key(tools, args_key, effective_args_key, call_id_key, gen_call_id_key);934    } else {935        auto name_spec = parse_key_spec(effective_name_key);936        auto args_spec = parse_key_spec(effective_args_key);937        if (!name_spec.first.empty() || !args_spec.first.empty()) {938            tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key);939        } else {940            tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper);941        }942    }943 944    // Build the section with markers945    auto tool_calls = tool_choices;946    if (parallel_tool_calls) {947        tool_calls = tool_calls + zero_or_more(space() + literal(",") + space() + tool_choices);948    }949 950    if (array_wrapped) {951        tool_calls = literal("[") + space() + tool_calls + space() + literal("]");952    }953 954    auto section =955        trigger_rule("tool-call", literal(section_start) + space() + tool_calls + space() + literal(section_end));956 957    return force_tool_calls ? section : optional(section);958}959 960void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) {961    for (const auto & node : result.nodes) {962        visit(arena, node);963    }964}965 966static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {967    const auto & node = arena.get(id);968 969    if (node.text.empty()) {970        return "";971    }972 973    if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {974        return std::string(node.text);975    }976 977    if (node.rule == "gemma4-string-content") {978        return escape_json_string_inner(std::string(node.text));979    }980 981    if (node.rule == "gemma4-string") {982        std::string result = "\"";983        if (!node.children.empty()) {984            result += gemma4_to_json(arena, node.children[0]);985            if (!node.is_partial) {986                result += "\"";987            }988        }989        return result;990    }991 992    if (node.rule == "gemma4-array") {993        std::string result = "[";994 995        bool add_comma = false;996        for (auto child_id : node.children) {997            if (add_comma) {998                result += ',';999            }1000            add_comma = true;1001            result += gemma4_to_json(arena, child_id);1002        }1003 1004        if (!node.is_partial) {1005            result += ']';1006        }1007        return result;1008    }1009 1010    if (node.rule == "gemma4-dict-key-name") {1011        return std::string(node.text);1012    }1013 1014    if (node.rule == "gemma4-dict-key") {1015        std::string result = "\"";1016        if (!node.children.empty()) {1017            result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));1018        }1019        if (!node.is_partial) {1020            result += "\":";1021        }1022        return result;1023    }1024 1025    if (node.rule == "gemma4-dict-kv") {1026        std::string result;1027        for (auto child_id : node.children) {1028            result += gemma4_to_json(arena, child_id);1029        }1030        return result;1031    }1032 1033    if (node.rule == "gemma4-dict") {1034        std::string result = "{";1035 1036        bool add_comma = false;1037        for (auto child_id : node.children) {1038            if (add_comma) {1039                result += ',';1040            }1041            add_comma = true;1042            result += gemma4_to_json(arena, child_id);1043        }1044 1045        if (!node.is_partial) {1046            result += '}';1047        }1048        return result;1049    }1050 1051    if (node.rule == "gemma4-value") {1052        if (!node.children.empty()) {1053            return gemma4_to_json(arena, node.children[0]);1054        }1055        return "";1056    }1057 1058    return "";1059}1060 1061void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {1062    const auto & node = arena.get(id);1063 1064    if (node.tag == "reasoning") {1065        result.reasoning_content += std::string(node.text);1066        return;1067    }1068 1069    if (node.tag == "content") {1070        result.content += std::string(node.text);1071        return;1072    }1073 1074    if (node.tag == "tool") {1075        auto name_id = arena.find_by_tag(node, "tool-name");1076        auto args_id = arena.find_by_tag(node, "tool-args");1077 1078        if (name_id != COMMON_PEG_INVALID_AST_ID && args_id != COMMON_PEG_INVALID_AST_ID) {1079            const auto & name_node = arena.get(name_id);1080            const auto & args_node = arena.get(args_id);1081 1082            if (!name_node.is_partial) {1083                common_chat_tool_call call;1084                call.name = std::string(name_node.text);1085                if (!args_node.children.empty()) {1086                    call.arguments = gemma4_to_json(arena, args_node.children[0]);1087                }1088                result.tool_calls.push_back(call);1089            }1090        }1091 1092        return;1093    }1094 1095    for (auto child_id : node.children) {1096        visit(arena, child_id);1097    }1098}1099 1100static void minimax_m3_collect(const common_peg_ast_arena &     arena,1101                               const common_peg_ast_node &      node,1102                               const std::string &              tag,1103                               std::vector<common_peg_ast_id> & out) {1104    for (auto child_id : node.children) {1105        const auto & child = arena.get(child_id);1106        if (child.tag == tag) {1107            out.push_back(child_id);1108        } else {1109            minimax_m3_collect(arena, child, tag, out);1110        }1111    }1112}1113 1114static common_peg_ast_id minimax_m3_value_of(const common_peg_ast_arena & arena, const common_peg_ast_node & node) {1115    for (auto child_id : node.children) {1116        const auto & tag = arena.get(child_id).tag;1117        if (tag == common_chat_peg_builder::TOOL_ARG_VALUE ||1118            tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE ||1119            tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT ||1120            tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) {1121            return child_id;1122        }1123    }1124    return COMMON_PEG_INVALID_AST_ID;1125}1126 1127static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed);1128 1129static std::string minimax_m3_member_to_json(const common_peg_ast_arena & arena, const common_peg_ast_node & node) {1130    auto name_id = arena.find_by_tag(node, common_chat_peg_builder::TOOL_ARG_NAME);1131    if (name_id == COMMON_PEG_INVALID_AST_ID) {1132        return "";1133    }1134 1135    return ordered_json(arena.get(name_id).text).dump() + ":" +1136           minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, node), !node.is_partial);1137}1138 1139static std::string minimax_m3_container_to_json(const common_peg_ast_arena & arena,1140                                                const common_peg_ast_node & node,1141                                                bool                        is_object,1142                                                bool                        closed) {1143    const std::string tag = is_object ? common_chat_peg_builder::TOOL_ARG1144                                      : common_chat_peg_minimax_m3_mapper::TOOL_ARG_ITEM;1145 1146    std::vector<common_peg_ast_id> entries;1147    minimax_m3_collect(arena, node, tag, entries);1148 1149    std::string result = is_object ? "{" : "[";1150 1151    bool add_comma = false;1152    for (auto entry_id : entries) {1153        const auto & entry = arena.get(entry_id);1154 1155        std::string text;1156        if (is_object) {1157            text = minimax_m3_member_to_json(arena, entry);1158        } else {1159            text = minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, entry), !entry.is_partial);1160        }1161 1162        if (text.empty()) {1163            continue;1164        }1165 1166        if (add_comma) {1167            result += ",";1168        }1169        add_comma = true;1170        result += text;1171    }1172 1173    if (closed) {1174        result += is_object ? "}" : "]";1175    }1176    return result;1177}1178 1179static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed) {1180    if (id == COMMON_PEG_INVALID_AST_ID) {1181        return "";1182    }1183 1184    const auto & node = arena.get(id);1185 1186    if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT) {1187        return minimax_m3_container_to_json(arena, node, /* is_object = */ true, closed);1188    }1189 1190    if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) {1191        return minimax_m3_container_to_json(arena, node, /* is_object = */ false, closed);1192    }1193 1194    if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) {1195        return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : "");1196    }1197 1198    // Numbers and booleans are written verbatim by the template1199    return std::string(node.text);1200}

Showing the first 1,200 of 1237 lines. Download the file for the rest.