echodict/llama.cpp

1#pragma once2 3#include "common.h"4#include "log.h"5#include "llama.h"6#include "chat.h"7#include "mtmd.h"8 9#define JSON_ASSERT GGML_ASSERT10#include <nlohmann/json.hpp>11 12#include <string>13#include <vector>14#include <cinttypes>15 16using json = nlohmann::ordered_json;17 18#define SLT_INF(slot, fmt, ...) LOG_INF("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)19#define SLT_CNT(slot, fmt, ...) LOG_CNT(""                                 fmt,                                                                __VA_ARGS__)20#define SLT_WRN(slot, fmt, ...) LOG_WRN("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)21#define SLT_ERR(slot, fmt, ...) LOG_ERR("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)22#define SLT_DBG(slot, fmt, ...) LOG_DBG("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)23 24#define SRV_INF(fmt, ...) LOG_INF("srv  %12.*s: " fmt, 12, __func__, __VA_ARGS__)25#define SRV_CNT(fmt, ...) LOG_CNT(""              fmt,               __VA_ARGS__)26#define SRV_WRN(fmt, ...) LOG_WRN("srv  %12.*s: " fmt, 12, __func__, __VA_ARGS__)27#define SRV_ERR(fmt, ...) LOG_ERR("srv  %12.*s: " fmt, 12, __func__, __VA_ARGS__)28#define SRV_DBG(fmt, ...) LOG_DBG("srv  %12.*s: " fmt, 12, __func__, __VA_ARGS__)29 30using raw_buffer = std::vector<uint8_t>;31 32template <typename T>33static T json_value(const json & body, const std::string & key, const T & default_value) {34    // Fallback null to default value35    if (body.contains(key) && !body.at(key).is_null()) {36        try {37            return body.at(key);38        } catch (NLOHMANN_JSON_NAMESPACE::detail::type_error const & err) {39            LOG_WRN("Wrong type supplied for parameter '%s'. Expected '%s', using default value: %s\n", key.c_str(), json(default_value).type_name(), err.what());40            return default_value;41        }42    } else {43        return default_value;44    }45}46 47// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/1148enum error_type {49    ERROR_TYPE_INVALID_REQUEST,50    ERROR_TYPE_AUTHENTICATION,51    ERROR_TYPE_SERVER,52    ERROR_TYPE_NOT_FOUND,53    ERROR_TYPE_PERMISSION,54    ERROR_TYPE_UNAVAILABLE, // custom error55    ERROR_TYPE_NOT_SUPPORTED, // custom error56    ERROR_TYPE_EXCEED_CONTEXT_SIZE, // custom error57};58 59// thin wrapper around common_grammar_trigger with (de)serialization functions60struct server_grammar_trigger {61    common_grammar_trigger value;62 63    server_grammar_trigger() = default;64    server_grammar_trigger(const common_grammar_trigger & value) : value(value) {}65    server_grammar_trigger(const json & in) {66        value.type = (common_grammar_trigger_type) in.at("type").get<int>();67        value.value = in.at("value").get<std::string>();68        if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {69            value.token = (llama_token) in.at("token").get<int>();70        }71    }72 73    json to_json() const {74        json out {75            {"type", (int) value.type},76            {"value", value.value},77        };78        if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {79            out["token"] = (int) value.token;80        }81        return out;82    }83};84 85json format_error_response(const std::string & message, const enum error_type type);86 87//88// random string / id89//90 91std::string random_string();92std::string gen_chatcmplid();93std::string gen_tool_call_id();94 95// get a random marker; note: each time the server restarts, the marker will be different96const char * get_media_marker();97 98//99// lora utils100//101 102// check whether the given lora set has only aloras activated (empty => false)103bool lora_all_alora(const std::vector<common_adapter_lora_info> & loras);104 105// if the two sets of loras are different, they require a cache clear unless the106// change is only from aloras to aloras.107bool lora_should_clear_cache(108        const std::vector<common_adapter_lora_info> & current,109        const std::vector<common_adapter_lora_info> & next);110 111std::map<int, float> parse_lora_request(const json & data);112 113bool are_lora_equal(114        const std::vector<common_adapter_lora_info> & l1,115        const std::vector<common_adapter_lora_info> & l2);116 117// get the ids of all enabled loras118std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras);119 120//121// server_tokens122//123 124/**125 * server_tokens is a helper to manage the input tokens and image for the server.126 * it is made this way to simplify the logic of KV cache management.127 */128struct server_tokens {129    bool has_mtmd = false;130 131private: // disallow accessing these members directly, risking out-of-sync132 133    // map a **start** index in tokens to the image chunk134    // note: the order need to be in-sync with tokens135    std::map<size_t, mtmd::input_chunk_ptr> map_idx_to_media;136 137    // list of tokens138    //   if the token is LLAMA_TOKEN_NULL, it indicates that this position is occupied by media chunk139    //   otherwise, it is a normal text token140    // note: a non-text chunk can occupy multiple tokens (aka memory cells) in the token list141    // note(2): for M-RoPE, an image can occupy different number of pos; do not assume 1-to-1 mapping tokens <-> pos142    llama_tokens tokens;143 144    // for ex. with input of 5 text tokens and 2 images (each image occupies 3 tokens and 2 pos):145    //      [0] [1] [2] [3] [4] [img0] [img0] [img0] [img1] [img1] [img1]146    // idx  0   1   2   3   4   5      6      7      8      9      10147    // pos  0   1   2   3   4   5      5      5      7      7      7148    // map_idx_to_media will contain: {5, img0}, {8, img1}149 150public:151    server_tokens() = default;152    ~server_tokens() = default;153 154    // Prevent copying155    // TODO: server_tokens should be copyable - remove this:156    server_tokens(const server_tokens&) = delete;157    server_tokens& operator=(const server_tokens&) = delete;158 159    // Allow moving (usually implicitly generated if members are movable)160    server_tokens(server_tokens&&) = default;161    server_tokens& operator=(server_tokens&&) = default;162 163    // Allow accessing elements using [] operator164    llama_token operator[](size_t index) { return tokens[index]; }165    const llama_token& operator[](size_t index) const { return tokens[index]; }166 167    server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);168    server_tokens(const llama_tokens & tokens, bool has_mtmd);169 170    // for debugging171    std::string str() const;172 173    // the next position after n_tokens. if n_tokens < 0, return the next position after all tokens.174    llama_pos pos_next(int64_t n_tokens = -1) const;175 176    // number of tokens with position < max_pos177    size_t size_up_to_pos(llama_pos max_pos) const;178 179    const mtmd::input_chunk_ptr & find_chunk(size_t idx) const;180 181    void push_back(llama_token tok);182 183    // will create a copy of the chunk if it contains non-text data184    void push_back(const mtmd_input_chunk * chunk);185 186    // appends server tokens, updates the media map. copies media chunks.187    void push_back(server_tokens & tokens);188 189    // for compatibility with context shift and prompt truncation190    void insert(const llama_tokens & inp_tokens);191 192    // for compatibility with speculative decoding, ctx shift, slot save/load193    const llama_tokens & get_tokens() const;194 195    llama_tokens get_text_tokens() const;196 197    // for compatibility with speculative decoding198    void set_token(llama_pos pos, llama_token id);199 200    size_t size() const { return tokens.size(); }201 202    bool empty() const { return tokens.empty(); }203 204    void clear() {205        map_idx_to_media.clear();206        tokens.clear();207    }208 209    void keep_first(size_t n);210 211    std::string detokenize(const llama_context * ctx, bool special) const;212 213    size_t get_common_prefix(const server_tokens & b) const;214 215    // make sure all text tokens are within the vocab range216    bool validate(const struct llama_context * ctx) const;217 218    // encode and decode the image chunk219    int32_t process_chunk(220                llama_context * ctx,221                mtmd_context * mctx,222                size_t idx,223                llama_pos pos,224                int32_t seq_id,225                size_t & n_tokens_out) const;226 227    server_tokens clone() const;228};229 230 231//232// tokenizer and input processing utils233//234 235bool json_is_array_of_numbers(const json & data);236 237// is array having BOTH numbers & strings?238bool json_is_array_of_mixed_numbers_strings(const json & data);239 240// does array have any individual integers/tokens?241bool json_is_array_and_contains_numbers(const json & data);242 243// get value by path(key1 / key2)244json json_get_nested_values(const std::vector<std::string> & paths, const json & js);245 246/**247 * this handles 2 cases:248 * - only string, example: "string"249 * - mixed string and tokens, example: [12, 34, "string", 56, 78]250 */251llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special);252 253// return the last index of character that can form a valid string254// if the last character is potentially cut in half, return the index before the cut255// if validate_utf8(text) == text.size(), then the whole text is valid utf8256size_t validate_utf8(const std::string& text);257 258// process mtmd prompt, return the server_tokens containing both text tokens and media chunks259server_tokens process_mtmd_prompt(mtmd_context * mctx, std::string prompt, std::vector<raw_buffer> files);260 261/**262 * break the input "prompt" object into multiple prompt if needed, then tokenize them263 * this supports these cases:264 * - "prompt": "string"265 * - "prompt": [12, 34, 56]266 * - "prompt": [12, 34, "string", 56, 78]267 * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }268 * and multiple prompts (multi-tasks):269 * - "prompt": ["string1", "string2"]270 * - "prompt": ["string1", [12, 34, 56]]271 * - "prompt": [[12, 34, 56], [78, 90, 12]]272 * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56], { "prompt_string": "string", "multimodal_data": [ "base64" ]}]273 */274std::vector<server_tokens> tokenize_input_prompts(275                                        const llama_vocab * vocab,276                                        mtmd_context * mctx,277                                        const json & json_prompt,278                                        bool add_special,279                                        bool parse_special);280 281//282// OAI utils283//284 285// global server parameters for chat formatting / parsing286struct server_chat_params {287    bool use_jinja;288    bool prefill_assistant;289    common_reasoning_format reasoning_format;290    std::map<std::string, std::string> chat_template_kwargs; // mapping key --> json value291    common_chat_templates_ptr tmpls;292    bool allow_image;293    bool allow_audio;294    bool enable_thinking = true;295    int  reasoning_budget = -1;296    std::string reasoning_budget_message;297    std::string media_path;298    bool force_pure_content = false;299};300 301// used by /completions endpoint302json oaicompat_completion_params_parse(const json & body);303 304// used by /chat/completions endpoint305json oaicompat_chat_params_parse(306    json & body, /* openai api json semantics */307    const server_chat_params & opt,308    std::vector<raw_buffer> & out_files);309 310// convert OpenAI Responses API format to OpenAI Chat Completions API format311json convert_responses_to_chatcmpl(const json & body);312 313// convert OpenAI transcriptions API format to OpenAI Chat Completions API format314json convert_transcriptions_to_chatcmpl(315    const json & body,316    const std::map<std::string, raw_buffer> & in_files,317    std::vector<raw_buffer> & out_files);318 319// convert Anthropic Messages API format to OpenAI Chat Completions API format320json convert_anthropic_to_oai(const json & body);321 322// TODO: move it to server-task.cpp323json format_embeddings_response_oaicompat(324    const json & request,325    const std::string & model_name,326    const json & embeddings,327    bool use_base64 = false);328 329// TODO: move it to server-task.cpp330json format_response_rerank(331        const json & request,332        const std::string & model_name,333        const json & ranks,334        bool is_tei_format,335        std::vector<std::string> & texts,336        int top_n);337 338//339// other utils340//341 342std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx);343 344std::string safe_json_to_str(const json & data);345 346std::string tokens_to_str(llama_context * ctx, const llama_tokens & tokens);347std::string tokens_to_str(const llama_vocab * vocab, const llama_tokens & tokens);348 349// format incomplete utf-8 multibyte character for output350std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token);351 352// format server-sent event (SSE), return the formatted string to send353// note: if data is a json array, it will be sent as multiple events, one per item354std::string format_oai_sse(const json & data);355 356std::string format_oai_resp_sse(const json & data);357 358// format Anthropic-style SSE with event types359std::string format_anthropic_sse(const json & data);360 361bool is_valid_utf8(const std::string & str);362 363//364// formatting output responses365// TODO: move these to server-task.cpp366//367 368llama_tokens format_prompt_infill(369        const llama_vocab * vocab,370        const json & input_prefix,371        const json & input_suffix,372        const json & input_extra,373        const int n_batch,374        const int n_predict,375        const int n_ctx,376        const bool spm_infill,377        const llama_tokens & tokens_prompt);378 379// format rerank task: [BOS]query[EOS][SEP]doc[EOS].380server_tokens format_prompt_rerank(381        const struct llama_model * model,382        const struct llama_vocab * vocab,383        mtmd_context * mctx,384        const std::string & query,385        const std::string & doc);386