Brunobkr/llama.cpp_AlgMor24_github

1// Chat support (incl. tool call grammar constraining & output parsing) w/ generic & custom template handlers.2 3#pragma once4 5#include "common.h"6#include "peg-parser.h"7#include "jinja/parser.h"8#include "jinja/runtime.h"9#include "jinja/caps.h"10 11#include "nlohmann/json_fwd.hpp"12 13#include <chrono>14#include <functional>15#include <map>16#include <string>17#include <vector>18 19using chat_template_caps = jinja::caps;20using json = nlohmann::ordered_json;21 22struct common_chat_templates;23 24namespace autoparser {25struct generation_params;26}  // namespace autoparser27 28struct common_chat_tool_call {29    std::string name;30    std::string arguments;31    std::string id;32 33    bool operator==(const common_chat_tool_call & other) const {34        return name == other.name && arguments == other.arguments && id == other.id;35    }36};37 38struct common_chat_msg_content_part {39    std::string type;40    std::string text;41 42    // TODO @ngxson : no known chat templates support reasoning_content in content parts yet43    //                this can be useful for models with interleaved thinking (like Kimi-K2)44    //                if you see any templates explicitly support this, please ping me45    // std::string reasoning_content;46 47    bool operator==(const common_chat_msg_content_part & other) const {48        return type == other.type && text == other.text;49    }50};51 52struct common_chat_template {53    jinja::program prog;54    std::string bos_tok;55    std::string eos_tok;56    std::string src;57    chat_template_caps caps;58 59    common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {60        jinja::lexer lexer;61        auto lexer_res = lexer.tokenize(src);62        this->prog = jinja::parse_from_tokens(lexer_res);63 64        this->src = lexer_res.source;65        this->bos_tok = bos_token;66        this->eos_tok = eos_token;67 68        this->caps = jinja::caps_get(prog);69        // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());70    }71 72    const std::string & source() const { return src; }73    const std::string & bos_token() const { return bos_tok; }74    const std::string & eos_token() const { return eos_tok; }75 76    chat_template_caps original_caps() const {77        return caps;78    }79};80 81struct common_chat_msg {82    std::string                               role;83    std::string                               content;84    std::vector<common_chat_msg_content_part> content_parts;85    std::vector<common_chat_tool_call>        tool_calls;86    std::string                               reasoning_content;87    std::string                               tool_name;88    std::string                               tool_call_id;89 90    nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const;91 92    std::string render_content(const std::string & delimiter = "\n\n") const;93 94    bool empty() const {95        return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() &&96               tool_name.empty() && tool_call_id.empty();97    }98 99    bool contains_media() const {100        for (const auto & part : content_parts) {101            if (part.type == "media_marker") {102                return true;103            }104        }105        return false;106    }107 108    void set_tool_call_ids(std::vector<std::string> &           ids_cache,109                           const std::function<std::string()> & gen_tool_call_id) {110        for (auto i = 0u; i < tool_calls.size(); i++) {111            if (ids_cache.size() <= i) {112                auto id = tool_calls[i].id;113                if (id.empty()) {114                    id = gen_tool_call_id();115                }116                ids_cache.push_back(id);117            }118            tool_calls[i].id = ids_cache[i];119        }120    }121 122    bool operator==(const common_chat_msg & other) const {123        return role == other.role && content == other.content && content_parts == other.content_parts &&124               tool_calls == other.tool_calls && reasoning_content == other.reasoning_content &&125               tool_name == other.tool_name && tool_call_id == other.tool_call_id;126    }127 128    bool operator!=(const common_chat_msg & other) const { return !(*this == other); }129};130 131struct common_chat_msg_diff {132    std::string           reasoning_content_delta;133    std::string           content_delta;134    size_t                tool_call_index = std::string::npos;135    common_chat_tool_call tool_call_delta;136 137    static std::vector<common_chat_msg_diff> compute_diffs(const common_chat_msg & msg_prv,138                                                           const common_chat_msg & msg_new);139 140    bool operator==(const common_chat_msg_diff & other) const {141        return content_delta == other.content_delta && tool_call_index == other.tool_call_index &&142               tool_call_delta == other.tool_call_delta;143    }144};145 146enum common_chat_role {147    COMMON_CHAT_ROLE_UNKNOWN,148    COMMON_CHAT_ROLE_SYSTEM,149    COMMON_CHAT_ROLE_ASSISTANT,150    COMMON_CHAT_ROLE_USER,151    COMMON_CHAT_ROLE_TOOL152};153 154common_chat_role common_chat_role_from_string(const std::string & role);155const char *     common_chat_role_to_string(common_chat_role role);156 157struct common_chat_msg_span {158    common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;159    std::size_t pos = 0;160    std::size_t len = 0;161 162    bool valid() const {163        return role != COMMON_CHAT_ROLE_UNKNOWN;164    }165};166 167struct common_chat_msg_spans {168    std::vector<common_chat_msg_span> spans;169 170    void add(common_chat_role role, size_t pos, size_t len) {171        spans.push_back({ role, pos, len });172    }173 174    bool is_user_start(int32_t pos) const {175        for (auto it = spans.begin(); it != spans.end(); ++it) {176            if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) {177                return true;178            }179        }180        return false;181    }182 183    int32_t last_user_message_pos() const {184        for (auto it = spans.rbegin(); it != spans.rend(); ++it) {185            if (it->role == COMMON_CHAT_ROLE_USER) {186                return (int32_t) it->pos;187            }188        }189        return -1;190    }191};192 193struct common_chat_msg_delimiter {194    common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;195    std::string      delimiter;196    llama_tokens     tokens = {};197};198 199struct common_chat_msg_delimiters {200    std::vector<common_chat_msg_delimiter> delimiters;201 202    common_chat_msg_delimiters() = default;203    common_chat_msg_delimiters(std::initializer_list<common_chat_msg_delimiter> delims) : delimiters(delims) {}204 205    void add(common_chat_role role, const std::string & delimiter) {206        delimiters.push_back({ role, delimiter });207    }208 209    void tokenize(const llama_vocab * vocab);210 211    // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching212    common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;213 214    nlohmann::ordered_json to_json() const;215};216 217struct common_chat_tool {218    std::string name;219    std::string description;220    std::string parameters;221};222 223enum common_chat_tool_choice {224    COMMON_CHAT_TOOL_CHOICE_AUTO,225    COMMON_CHAT_TOOL_CHOICE_REQUIRED,226    COMMON_CHAT_TOOL_CHOICE_NONE,227};228 229enum common_chat_format {230    COMMON_CHAT_FORMAT_CONTENT_ONLY,231 232    // These are intended to be parsed by the PEG parser233    COMMON_CHAT_FORMAT_PEG_SIMPLE,234    COMMON_CHAT_FORMAT_PEG_NATIVE,235    COMMON_CHAT_FORMAT_PEG_GEMMA4,236    COMMON_CHAT_FORMAT_PEG_MINIMAX_M3,237 238    COMMON_CHAT_FORMAT_COUNT,  // Not a format, just the # formats239};240 241 242// Continuation method provided via `continue_final_message`243enum common_chat_continuation {244    COMMON_CHAT_CONTINUATION_NONE,245    COMMON_CHAT_CONTINUATION_AUTO,246    COMMON_CHAT_CONTINUATION_REASONING,247    COMMON_CHAT_CONTINUATION_CONTENT,248};249 250struct common_chat_templates_inputs {251    std::vector<common_chat_msg>          messages;252    std::string                           grammar;253    std::string                           json_schema;254    bool                                  add_generation_prompt  = true;255    common_chat_continuation              continue_final_message = COMMON_CHAT_CONTINUATION_NONE;256    bool                                  use_jinja              = true;257    // Parameters below only supported when use_jinja is true258    std::vector<common_chat_tool>         tools;259    common_chat_tool_choice               tool_choice         = COMMON_CHAT_TOOL_CHOICE_AUTO;260    bool                                  parallel_tool_calls = false;261    common_reasoning_format               reasoning_format    = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool enable_thinking"262    bool                                  enable_thinking     = true;263    std::chrono::system_clock::time_point now                 = std::chrono::system_clock::now();264    std::map<std::string, std::string>    chat_template_kwargs;265    bool                                  add_bos = false;266    bool                                  add_eos = false;267    bool                                  force_pure_content = false;268};269 270struct common_chat_params {271    common_chat_format                  format = COMMON_CHAT_FORMAT_CONTENT_ONLY;272    std::string                         prompt;273    std::string                         grammar;274    bool                                grammar_lazy         = false;275    std::string                         generation_prompt;276    bool                                supports_thinking    = false;277    std::string                         thinking_start_tag;  // e.g., "<think>"278    std::vector<std::string>            thinking_end_tags;   // e.g., "</think>"279    std::vector<common_grammar_trigger> grammar_triggers;280    std::vector<std::string>            preserved_tokens;281    std::vector<std::string>            additional_stops;282    std::string                         parser;283    common_chat_msg_delimiters          message_delimiters;284};285 286// per-message parsing syntax287// should be derived from common_chat_params288struct common_chat_parser_params {289    common_chat_format      format               = COMMON_CHAT_FORMAT_CONTENT_ONLY;290    common_reasoning_format reasoning_format     = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"291    // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)292    bool                    reasoning_in_content = false;293    std::string             generation_prompt;294    bool                    parse_tool_calls     = true;295    bool                    is_continuation      = false;296    bool                    echo                 = false;  // Include assistant prefilled msg in output297    bool                    debug                = false;  // Enable debug output for PEG parser298    common_peg_arena        parser               = {};299    common_chat_parser_params() = default;300    common_chat_parser_params(const common_chat_params & chat_params) {301        format  = chat_params.format;302        generation_prompt = chat_params.generation_prompt;303    }304};305 306// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid307bool common_chat_verify_template(const std::string & tmpl, bool use_jinja);308 309void common_chat_templates_free(struct common_chat_templates * tmpls);310 311struct common_chat_templates_deleter {312    void operator()(common_chat_templates * tmpls) { common_chat_templates_free(tmpls); }313};314 315typedef std::unique_ptr<struct common_chat_templates, common_chat_templates_deleter> common_chat_templates_ptr;316 317common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,318                                                     const std::string &        chat_template_override,319                                                     const std::string &        bos_token_override = "",320                                                     const std::string &        eos_token_override = "");321 322bool        common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);323std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = "");324 325struct common_chat_params common_chat_templates_apply(const struct common_chat_templates *        tmpls,326                                                      const struct common_chat_templates_inputs & inputs);327 328// Format single message, while taking into account the position of that message in chat history329std::string common_chat_format_single(const struct common_chat_templates * tmpls,330                                      const std::vector<common_chat_msg> & past_msg,331                                      const common_chat_msg &              new_msg,332                                      bool                                 add_ass,333                                      bool                                 use_jinja);334 335// Returns an example of formatted chat336std::string common_chat_format_example(const struct common_chat_templates *       tmpls,337                                       bool                                       use_jinja,338                                       const std::map<std::string, std::string> & chat_template_kwargs);339 340const char *    common_chat_format_name(common_chat_format format);341common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);342common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);343 344// used by arg and server345const char *            common_reasoning_format_name(common_reasoning_format format);346common_reasoning_format common_reasoning_format_from_name(const std::string & format);347 348common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);349 350bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);351 352// Parses a JSON array of messages in OpenAI's chat completion API format.353std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const nlohmann::ordered_json & messages);354 355std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const nlohmann::ordered_json & tools);356 357common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value);358 359// DEPRECATED: only used in tests360nlohmann::ordered_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);361 362nlohmann::ordered_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);363 364// get template caps, useful for reporting to server /props endpoint365std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);366 367std::string common_chat_template_direct_apply(368    const common_chat_template & tmpl,369    const autoparser::generation_params & inputs);370 371std::string common_chat_template_generation_prompt(372    const common_chat_template &          tmpl,373    const autoparser::generation_params & inputs);374 375std::optional<common_chat_params> common_chat_try_specialized_template(376        const common_chat_template &          tmpl,377        const std::string &                   src,378        autoparser::generation_params & params);379 380 381// specialized per-task preset382struct common_chat_prompt_preset {383    std::string system;384    std::string user;385};386 387common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);388 389common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters);390