echodict/llama.cpp

1// Chat support (incl. tool call grammar constraining & output parsing) w/ generic & custom template handlers.2 3#pragma once4 5#include "common.h"6#include "peg-parser.h"7#include "jinja/parser.h"8#include "jinja/runtime.h"9#include "jinja/caps.h"10 11#include "nlohmann/json_fwd.hpp"12 13#include <chrono>14#include <functional>15#include <map>16#include <string>17#include <vector>18 19using chat_template_caps = jinja::caps;20using json = nlohmann::ordered_json;21 22struct common_chat_templates;23 24namespace autoparser {25struct generation_params;26}  // namespace autoparser27 28struct common_chat_tool_call {29    std::string name;30    std::string arguments;31    std::string id;32 33    bool operator==(const common_chat_tool_call & other) const {34        return name == other.name && arguments == other.arguments && id == other.id;35    }36};37 38struct common_chat_msg_content_part {39    std::string type;40    std::string text;41 42    // TODO @ngxson : no known chat templates support reasoning_content in content parts yet43    //                this can be useful for models with interleaved thinking (like Kimi-K2)44    //                if you see any templates explicitly support this, please ping me45    // std::string reasoning_content;46 47    bool operator==(const common_chat_msg_content_part & other) const {48        return type == other.type && text == other.text;49    }50};51 52struct common_chat_template {53    jinja::program prog;54    std::string bos_tok;55    std::string eos_tok;56    std::string src;57    chat_template_caps caps;58 59    common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {60        jinja::lexer lexer;61        auto lexer_res = lexer.tokenize(src);62        this->prog = jinja::parse_from_tokens(lexer_res);63 64        this->src = lexer_res.source;65        this->bos_tok = bos_token;66        this->eos_tok = eos_token;67 68        this->caps = jinja::caps_get(prog);69        // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());70    }71 72    const std::string & source() const { return src; }73    const std::string & bos_token() const { return bos_tok; }74    const std::string & eos_token() const { return eos_tok; }75 76    chat_template_caps original_caps() const {77        return caps;78    }79};80 81struct common_chat_msg {82    std::string                               role;83    std::string                               content;84    std::vector<common_chat_msg_content_part> content_parts;85    std::vector<common_chat_tool_call>        tool_calls;86    std::string                               reasoning_content;87    std::string                               tool_name;88    std::string                               tool_call_id;89 90    nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const;91 92    bool empty() const {93        return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() &&94               tool_name.empty() && tool_call_id.empty();95    }96 97    void set_tool_call_ids(std::vector<std::string> &           ids_cache,98                           const std::function<std::string()> & gen_tool_call_id) {99        for (auto i = 0u; i < tool_calls.size(); i++) {100            if (ids_cache.size() <= i) {101                auto id = tool_calls[i].id;102                if (id.empty()) {103                    id = gen_tool_call_id();104                }105                ids_cache.push_back(id);106            }107            tool_calls[i].id = ids_cache[i];108        }109    }110 111    bool operator==(const common_chat_msg & other) const {112        return role == other.role && content == other.content && content_parts == other.content_parts &&113               tool_calls == other.tool_calls && reasoning_content == other.reasoning_content &&114               tool_name == other.tool_name && tool_call_id == other.tool_call_id;115    }116 117    bool operator!=(const common_chat_msg & other) const { return !(*this == other); }118};119 120struct common_chat_msg_diff {121    std::string           reasoning_content_delta;122    std::string           content_delta;123    size_t                tool_call_index = std::string::npos;124    common_chat_tool_call tool_call_delta;125 126    static std::vector<common_chat_msg_diff> compute_diffs(const common_chat_msg & msg_prv,127                                                           const common_chat_msg & msg_new);128 129    bool operator==(const common_chat_msg_diff & other) const {130        return content_delta == other.content_delta && tool_call_index == other.tool_call_index &&131               tool_call_delta == other.tool_call_delta;132    }133};134 135struct common_chat_tool {136    std::string name;137    std::string description;138    std::string parameters;139};140 141enum common_chat_tool_choice {142    COMMON_CHAT_TOOL_CHOICE_AUTO,143    COMMON_CHAT_TOOL_CHOICE_REQUIRED,144    COMMON_CHAT_TOOL_CHOICE_NONE,145};146 147enum common_chat_format {148    COMMON_CHAT_FORMAT_CONTENT_ONLY,149 150    // These are intended to be parsed by the PEG parser151    COMMON_CHAT_FORMAT_PEG_SIMPLE,152    COMMON_CHAT_FORMAT_PEG_NATIVE,153    COMMON_CHAT_FORMAT_PEG_GEMMA4,154 155    COMMON_CHAT_FORMAT_COUNT,  // Not a format, just the # formats156};157 158struct common_chat_templates_inputs {159    std::vector<common_chat_msg>          messages;160    std::string                           grammar;161    std::string                           json_schema;162    bool                                  add_generation_prompt = true;163    bool                                  use_jinja             = true;164    // Parameters below only supported when use_jinja is true165    std::vector<common_chat_tool>         tools;166    common_chat_tool_choice               tool_choice         = COMMON_CHAT_TOOL_CHOICE_AUTO;167    bool                                  parallel_tool_calls = false;168    common_reasoning_format               reasoning_format    = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool enable_thinking"169    bool                                  enable_thinking     = true;170    std::chrono::system_clock::time_point now                 = std::chrono::system_clock::now();171    std::map<std::string, std::string>    chat_template_kwargs;172    bool                                  add_bos = false;173    bool                                  add_eos = false;174    bool                                  force_pure_content = false;175};176 177struct common_chat_params {178    common_chat_format                  format = COMMON_CHAT_FORMAT_CONTENT_ONLY;179    std::string                         prompt;180    std::string                         grammar;181    bool                                grammar_lazy         = false;182    std::string                         generation_prompt;183    bool                                supports_thinking    = false;184    std::string                         thinking_start_tag;  // e.g., "<think>"185    std::string                         thinking_end_tag;    // e.g., "</think>"186    std::vector<common_grammar_trigger> grammar_triggers;187    std::vector<std::string>            preserved_tokens;188    std::vector<std::string>            additional_stops;189    std::string                         parser;190};191 192// per-message parsing syntax193// should be derived from common_chat_params194struct common_chat_parser_params {195    common_chat_format      format               = COMMON_CHAT_FORMAT_CONTENT_ONLY;196    common_reasoning_format reasoning_format     = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"197    // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)198    bool                    reasoning_in_content = false;199    std::string             generation_prompt;200    bool                    parse_tool_calls     = true;201    bool                    debug                = false;  // Enable debug output for PEG parser202    common_peg_arena        parser               = {};203    common_chat_parser_params() = default;204    common_chat_parser_params(const common_chat_params & chat_params) {205        format  = chat_params.format;206        generation_prompt = chat_params.generation_prompt;207    }208};209 210// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid211bool common_chat_verify_template(const std::string & tmpl, bool use_jinja);212 213void common_chat_templates_free(struct common_chat_templates * tmpls);214 215struct common_chat_templates_deleter {216    void operator()(common_chat_templates * tmpls) { common_chat_templates_free(tmpls); }217};218 219typedef std::unique_ptr<struct common_chat_templates, common_chat_templates_deleter> common_chat_templates_ptr;220 221common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,222                                                     const std::string &        chat_template_override,223                                                     const std::string &        bos_token_override = "",224                                                     const std::string &        eos_token_override = "");225 226bool        common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);227std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = "");228 229struct common_chat_params common_chat_templates_apply(const struct common_chat_templates *        tmpls,230                                                      const struct common_chat_templates_inputs & inputs);231 232// Format single message, while taking into account the position of that message in chat history233std::string common_chat_format_single(const struct common_chat_templates * tmpls,234                                      const std::vector<common_chat_msg> & past_msg,235                                      const common_chat_msg &              new_msg,236                                      bool                                 add_ass,237                                      bool                                 use_jinja);238 239// Returns an example of formatted chat240std::string common_chat_format_example(const struct common_chat_templates *       tmpls,241                                       bool                                       use_jinja,242                                       const std::map<std::string, std::string> & chat_template_kwargs);243 244const char *    common_chat_format_name(common_chat_format format);245common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);246common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);247 248// used by arg and server249const char *            common_reasoning_format_name(common_reasoning_format format);250common_reasoning_format common_reasoning_format_from_name(const std::string & format);251 252common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);253 254bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);255 256// Parses a JSON array of messages in OpenAI's chat completion API format.257std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const nlohmann::ordered_json & messages);258 259// DEPRECATED: only used in tests260nlohmann::ordered_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);261 262std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const nlohmann::ordered_json & tools);263nlohmann::ordered_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);264 265nlohmann::ordered_json common_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);266 267// get template caps, useful for reporting to server /props endpoint268std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);269 270std::string common_chat_template_direct_apply(271    const common_chat_template & tmpl,272    const autoparser::generation_params & inputs);273 274std::optional<common_chat_params> common_chat_try_specialized_template(275        const common_chat_template &          tmpl,276        const std::string &                   src,277        autoparser::generation_params & params);278