Brunobkr/llama.cpp_AlgMor24_github

1#include "common.h"2#include "download.h"3#include "log.h"4#include "llama.h"5#include "mtmd.h"6#include "mtmd-helper.h"7#include "chat.h"8#include "base64.hpp"9 10#include "server-common.h"11 12#include <random>13#include <sstream>14#include <fstream>15#include <limits>16 17json format_error_response(const std::string & message, const enum error_type type) {18    std::string type_str;19    int code = 500;20    switch (type) {21        case ERROR_TYPE_INVALID_REQUEST:22            type_str = "invalid_request_error";23            code = 400;24            break;25        case ERROR_TYPE_AUTHENTICATION:26            type_str = "authentication_error";27            code = 401;28            break;29        case ERROR_TYPE_NOT_FOUND:30            type_str = "not_found_error";31            code = 404;32            break;33        case ERROR_TYPE_SERVER:34            type_str = "server_error";35            code = 500;36            break;37        case ERROR_TYPE_PERMISSION:38            type_str = "permission_error";39            code = 403;40            break;41        case ERROR_TYPE_NOT_SUPPORTED:42            type_str = "not_supported_error";43            code = 501;44            break;45        case ERROR_TYPE_UNAVAILABLE:46            type_str = "unavailable_error";47            code = 503;48            break;49        case ERROR_TYPE_EXCEED_CONTEXT_SIZE:50            type_str = "exceed_context_size_error";51            code = 400;52            break;53    }54    return json {55        {"code", code},56        {"message", message},57        {"type", type_str},58    };59}60 61//62// random string / id63//64 65std::string random_string() {66    static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");67 68    std::random_device rd;69    std::mt19937 generator(rd());70 71    std::string result(32, ' ');72 73    for (int i = 0; i < 32; ++i) {74        result[i] = str[generator() % str.size()];75    }76 77    return result;78}79 80std::string gen_chatcmplid() {81    return "chatcmpl-" + random_string();82}83 84std::string gen_tool_call_id() {85    return random_string();86}87 88const char * get_media_marker() {89    static const std::string marker = []() {90        // allow user to pin a reproducible marker via env var91        const char * env = getenv("LLAMA_MEDIA_MARKER");92        if (env && env[0] != '\0') {93            return std::string(env);94        }95        return std::string("<__media_") + random_string() + "__>";96    }();97    return marker.c_str();98}99 100//101// lora utils102//103 104bool lora_all_alora(const std::vector<common_adapter_lora_info> & loras) {105    bool found_alora = false;106    for (const auto & lora : loras) {107        if (lora.scale != 0) {108            if (llama_adapter_get_alora_n_invocation_tokens(lora.ptr) == 0) {109                return false;110            }111            found_alora = true;112        }113    }114    return found_alora;115}116 117bool lora_should_clear_cache(118        const std::vector<common_adapter_lora_info> & current,119        const std::vector<common_adapter_lora_info> & next) {120 121    // This should always be called after determining that the two sets are122    // _not_ equal. This assert is therefore some slightly wasted work and123    // should be safe to remove as long as this method is called correctly.124    GGML_ASSERT(!are_lora_equal(current, next));125 126    return (127        !(lora_get_enabled_ids(current).empty() || lora_all_alora(current)) ||128        !lora_all_alora(next));129}130 131std::map<int, float> parse_lora_request(const json & data) {132    std::map<int, float> lora;133 134    // set value135    for (const auto & entry : data) {136        int id      = json_value(entry, "id", -1);137        float scale = json_value(entry, "scale", 0.0f);138        lora[id] = scale;139    }140 141    return lora;142}143 144bool are_lora_equal(145        const std::vector<common_adapter_lora_info> & l1,146        const std::vector<common_adapter_lora_info> & l2) {147    if (l1.size() != l2.size()) {148        return false;149    }150    for (size_t i = 0; i < l1.size(); ++i) {151        // we don't check lora.path to reduce the time complexity152        if (l1[i].scale != l2[i].scale || l1[i].ptr != l2[i].ptr) {153            return false;154        }155    }156    return true;157}158 159std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras) {160    std::vector<size_t> enabled_ids;161    for (size_t i = 0; i < loras.size(); ++i) {162        if (loras[i].scale > 0) {163            enabled_ids.push_back(i);164        }165    }166    return enabled_ids;167}168 169//170// base64 utils (TODO: use the base64::decode from base64.hpp)171//172 173static const std::string base64_chars =174             "ABCDEFGHIJKLMNOPQRSTUVWXYZ"175             "abcdefghijklmnopqrstuvwxyz"176             "0123456789+/";177 178static inline bool is_base64(uint8_t c) {179    return (isalnum(c) || (c == '+') || (c == '/'));180}181 182static inline raw_buffer base64_decode(const std::string & encoded_string) {183    int i = 0;184    int j = 0;185    int in_ = 0;186 187    int in_len = encoded_string.size();188 189    uint8_t char_array_4[4];190    uint8_t char_array_3[3];191 192    raw_buffer ret;193 194    while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {195        char_array_4[i++] = encoded_string[in_]; in_++;196        if (i == 4) {197            for (i = 0; i < 4; i++) {198                char_array_4[i] = base64_chars.find(char_array_4[i]);199            }200 201            char_array_3[0] = ((char_array_4[0]      ) << 2) + ((char_array_4[1] & 0x30) >> 4);202            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);203            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) +   char_array_4[3];204 205            for (i = 0; (i < 3); i++) {206                ret.push_back(char_array_3[i]);207            }208 209            i = 0;210        }211    }212 213    if (i) {214        for (j = i; j < 4; j++) {215            char_array_4[j] = 0;216        }217 218        for (j = 0; j < 4; j++) {219            char_array_4[j] = base64_chars.find(char_array_4[j]);220        }221 222        char_array_3[0] = ((char_array_4[0]      ) << 2) + ((char_array_4[1] & 0x30) >> 4);223        char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);224        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) +   char_array_4[3];225 226        for (j = 0; j < i - 1; j++) {227            ret.push_back(char_array_3[j]);228        }229    }230 231    return ret;232}233 234//235// server_tokens implementation236//237 238server_tokens::server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd) : has_mtmd(has_mtmd) {239    for (size_t i = 0; i < mtmd_chunks.size(); ++i) {240        push_back(mtmd_chunks[i]);241    }242}243 244server_tokens::server_tokens(const llama_tokens & tokens, bool has_mtmd) : has_mtmd(has_mtmd), tokens(tokens) {245}246 247llama_pos server_tokens::pos_next(int64_t n_tokens) const {248    if (!has_mtmd) {249        if (n_tokens < 0) {250            return tokens.size();251        }252 253        return n_tokens;254    }255 256    if (n_tokens < 0) {257        llama_pos res = tokens.size();258 259        for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {260            const auto & chunk = it->second;261            res += mtmd_input_chunk_get_n_pos(chunk.get()) - mtmd_input_chunk_get_n_tokens(chunk.get());262        }263 264        return res;265    }266 267    int64_t idx = 0;268    llama_pos pos = 0;269 270    GGML_ASSERT(n_tokens <= (int64_t)tokens.size());271 272    while (idx < n_tokens) {273        const auto media_it = map_idx_to_media.find(idx);274        if (media_it != map_idx_to_media.end()) {275            const auto & chunk = media_it->second;276            const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());277            const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());278 279            pos += n_pos;280            idx += n_tok;281        } else {282            pos++;283            idx++;284        }285    }286 287    return pos;288}289 290size_t server_tokens::size_up_to_pos(llama_pos max_pos) const {291    if (!has_mtmd) {292        return std::min((size_t)max_pos, tokens.size());293    }294 295    size_t idx = 0;296    llama_pos pos = 0;297 298    while (idx < tokens.size()) {299        const auto media_it = map_idx_to_media.find(idx);300        if (media_it != map_idx_to_media.end()) {301            const auto & chunk = media_it->second;302            const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());303            const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());304 305            pos += n_pos;306            idx += n_tok;307        } else {308            pos++;309            idx++;310        }311 312        if (pos >= max_pos) {313            break;314        }315    }316 317    return idx;318}319 320std::string server_tokens::str() const {321    std::ostringstream oss;322    oss << "tokens: ";323    for (size_t idx = 0; idx < tokens.size(); ++idx) {324        llama_token t = tokens[idx];325        oss << "idx:" << idx << " ";326        if (t == LLAMA_TOKEN_NULL) {327            oss << "<embd> ";328        } else {329            oss << t << " ";330        }331    }332    oss << "\n";333    oss << "image idx: ";334    for (const auto & it : map_idx_to_media) {335        oss << it.first << ", ";336    }337    return oss.str();338}339 340const mtmd::input_chunk_ptr & server_tokens::find_chunk(size_t idx) const {341    auto it = map_idx_to_media.find(idx);342    if (it != map_idx_to_media.end()) {343        return it->second;344    }345    throw std::runtime_error("Chunk not found");346}347 348std::pair<const mtmd::input_chunk_ptr *, size_t> server_tokens::find_next_media_chunk(size_t idx) const {349    auto it = map_idx_to_media.upper_bound(idx);350    if (it != map_idx_to_media.end()) {351        return { &it->second, it->first };352    }353    return { nullptr, 0 };354}355 356void server_tokens::push_back(llama_token tok) {357    if (tok == LLAMA_TOKEN_NULL) {358        throw std::runtime_error("Invalid token");359    }360    tokens.emplace_back(tok);361}362 363void server_tokens::push_back(const mtmd_input_chunk * chunk) {364    auto type = mtmd_input_chunk_get_type(chunk);365    if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {366        GGML_ASSERT(has_mtmd);367        const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);368        size_t start_idx = tokens.size();369        for (size_t i = 0; i < n_tokens; ++i) {370            tokens.emplace_back(LLAMA_TOKEN_NULL);371        }372        mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));373        map_idx_to_media[start_idx] = std::move(new_chunk);374    } else if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {375        size_t n_tokens;376        const auto * text_tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);377        for (size_t i = 0; i < n_tokens; ++i) {378            push_back(text_tokens[i]);379        }380    } else {381        GGML_ABORT("Invalid chunk type");382    }383}384 385void server_tokens::push_back(server_tokens & tokens) {386    size_t start_idx = size();387    for (size_t i = 0; i < tokens.size(); i++) {388        push_back(tokens[i]);389    }390    if (tokens.has_mtmd) {391        // Assert if we are copying MTMD chunks to a server_tokens that does not have mtmd.392        // We could also just check, but this will prevent silently dropping MTMD data.393        GGML_ASSERT(has_mtmd);394        for (auto it = tokens.map_idx_to_media.begin(); it != tokens.map_idx_to_media.end(); ) {395            auto * chunk = tokens.map_idx_to_media[it->first].get();396            mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));397            map_idx_to_media[start_idx + it->first] = std::move(new_chunk);398        }399    }400}401 402void server_tokens::insert(const llama_tokens & inp_tokens) {403    tokens.insert(tokens.end(), inp_tokens.begin(), inp_tokens.end());404}405 406const llama_tokens & server_tokens::get_tokens() const {407    GGML_ASSERT(!has_mtmd);408    return tokens;409}410 411llama_tokens server_tokens::get_text_tokens() const {412    llama_tokens res;413    res.reserve(tokens.size());414    for (llama_token t : tokens) {415        if (t != LLAMA_TOKEN_NULL) {416            res.push_back(t);417        }418    }419    return res;420}421 422void server_tokens::set_token(llama_pos pos, llama_token id) {423    GGML_ASSERT(!has_mtmd); // only allow this if mtmd is disabled424    tokens[pos] = id;425}426 427void server_tokens::keep_first(size_t n) {428    GGML_ASSERT(n <= tokens.size());429    if (has_mtmd) {430        if (n == tokens.size()) {431            return; // nothing to do432        }433        // we throw an error if we try to remove a token in the middle of an image434        // for ex. with input of 5 text tokens and 2 images:435        //    [0] [1] [2] [3] [4] [img0] [img0] [img0] [img1] [img1]436        // n  1   2   3   4   5   6      7      8      9      10437        // allowed to resize      ^                    ^438        // disallowed to resize          ^      ^             ^439        if (n > 0) {440            // make sure we never remove tokens in the middle of an image441            // note that the case where we keep a full image at the end is allowed:442            //   tokens[n - 1] == LLAMA_TOKEN_NULL && tokens[n] != LLAMA_TOKEN_NULL443            if (tokens[n - 1] == LLAMA_TOKEN_NULL && tokens[n] == LLAMA_TOKEN_NULL) {444                find_chunk(n - 1); // will throw an error if the token is not begin-of-chunk445            }446        }447        // remove all image chunks that are not used anymore448        for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ) {449            size_t idx = it->first;450            if (idx >= n) {451                it = map_idx_to_media.erase(it);452            } else {453                ++it;454            }455        }456    }457    tokens.resize(n);458}459 460std::string server_tokens::detokenize(const llama_context * ctx, bool special) const {461    llama_tokens text_tokens;462    text_tokens.reserve(tokens.size());463    for (const auto & t : tokens) {464        if (t != LLAMA_TOKEN_NULL) {465            text_tokens.push_back(t);466        }467    }468    return common_detokenize(ctx, text_tokens, special);469}470 471size_t server_tokens::get_common_prefix(const server_tokens & b) const {472    const size_t max_idx = std::min(tokens.size(), b.tokens.size());473 474    if (!has_mtmd) {475        for (size_t i = 0; i < max_idx; ++i) {476            if (tokens[i] == b.tokens[i]) {477                continue;478            }479 480            return i;481        }482 483        return max_idx;484    }485 486    for (size_t i = 0; i < max_idx; ++i) {487        const llama_token ai =   tokens[i];488        const llama_token bi = b.tokens[i];489 490        if (ai == LLAMA_TOKEN_NULL && bi == LLAMA_TOKEN_NULL) {491            const auto & a_chunk =   find_chunk(i);492            const auto & b_chunk = b.find_chunk(i);493 494            GGML_ASSERT(a_chunk && b_chunk);495 496            const std::string id_ai = mtmd_input_chunk_get_id(a_chunk.get());497            const std::string id_bi = mtmd_input_chunk_get_id(b_chunk.get());498 499            const size_t n_tok_a = mtmd_input_chunk_get_n_tokens(a_chunk.get());500            const size_t n_tok_b = mtmd_input_chunk_get_n_tokens(b_chunk.get());501 502            if (id_ai == id_bi && n_tok_a == n_tok_b) {503                GGML_ASSERT(n_tok_a > 0 && "Invalid media chunk"); // should never happen504                i += n_tok_a - 1; // will be +1 by the for loop505                continue;506            }507 508            return i;509        }510 511        if (ai == bi) {512            continue;513        }514 515        return i;516    }517 518    return max_idx; // all tokens are equal519}520 521common_chat_msg_spans server_tokens::find_message_spans(const common_chat_msg_delimiters & delims) const {522    std::map<size_t, size_t> skips;523    for (const auto & it : map_idx_to_media) {524        skips[it.first] = mtmd_input_chunk_get_n_tokens(it.second.get());525    }526    return delims.split(tokens, skips);527}528 529bool server_tokens::validate(const struct llama_context * ctx) const {530    const llama_model * model = llama_get_model(ctx);531    const llama_vocab * vocab = llama_model_get_vocab(model);532    const int32_t n_vocab = llama_vocab_n_tokens(vocab);533 534    for (size_t i = 0; i < tokens.size(); ++i) {535        const auto & t = tokens[i];536        if (t == LLAMA_TOKEN_NULL) {537            try {538                const auto & chunk = find_chunk(i);539                size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk.get());540                i += n_tokens - 1; // will be +1 by the for loop541            } catch (const std::exception & e) {542                return false;543            }544        } else if (t < 0 || t >= n_vocab) {545            return false;546        }547    }548    return true;549}550 551server_tokens server_tokens::clone() const {552    server_tokens res;553    res.has_mtmd = has_mtmd;554    res.tokens   = tokens;555    for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {556        size_t idx = it->first;557        const mtmd::input_chunk_ptr & chunk = it->second;558        res.map_idx_to_media[idx] = mtmd::input_chunk_ptr(mtmd_input_chunk_copy(chunk.get()));559    }560    return res;561}562 563//564// tokenizer and input processing utils565//566 567bool json_is_array_of_numbers(const json & data) {568    if (data.is_array()) {569        for (const auto & e : data) {570            if (!e.is_number_integer()) {571                return false;572            }573        }574        return true;575    }576    return false;577}578 579bool json_is_array_of_mixed_numbers_strings(const json & data) {580    bool seen_string = false;581    bool seen_number = false;582    if (data.is_array()) {583        for (const auto & e : data) {584            seen_string |= e.is_string();585            seen_number |= e.is_number_integer();586            if (seen_number && seen_string) {587                return true;588            }589        }590    }591    return false;592}593 594bool json_is_array_and_contains_numbers(const json & data) {595    if (data.is_array()) {596        for (const auto & e : data) {597            if (e.is_number_integer()) {598                return true;599            }600        }601        return false;602    }603    return false;604}605 606json json_get_nested_values(const std::vector<std::string> & paths, const json & js) {607    json result = json::object();608 609    for (const std::string & path : paths) {610        json current = js;611        const auto keys = string_split<std::string>(path, /*separator*/ '/');612        bool valid_path = true;613        for (const std::string & k : keys) {614            if (valid_path && current.is_object() && current.contains(k)) {615                current = current[k];616            } else {617                valid_path = false;618            }619        }620        if (valid_path) {621            result[path] = current;622        }623    }624    return result;625}626 627llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special) {628    // If `add_bos` is true, we only add BOS, when json_prompt is a string,629    // or the first element of the json_prompt array is a string.630    llama_tokens prompt_tokens;631 632    if (json_prompt.is_array()) {633        bool first = true;634        for (const auto & p : json_prompt) {635            if (p.is_string()) {636                auto s = p.template get<std::string>();637 638                llama_tokens p;639                if (first) {640                    p = common_tokenize(vocab, s, add_special, parse_special);641                    first = false;642                } else {643                    p = common_tokenize(vocab, s, false, parse_special);644                }645 646                prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());647            } else {648                if (first) {649                    first = false;650                }651 652                prompt_tokens.push_back(p.template get<llama_token>());653            }654        }655    } else {656        auto s = json_prompt.template get<std::string>();657        prompt_tokens = common_tokenize(vocab, s, add_special, parse_special);658    }659 660    return prompt_tokens;661}662 663size_t validate_utf8(const std::string& text) {664    size_t len = text.size();665    if (len == 0) return 0;666 667    // Check the last few bytes to see if a multi-byte character is cut off668    for (size_t i = 1; i <= 4 && i <= len; ++i) {669        unsigned char c = text[len - i];670        // Check for start of a multi-byte sequence from the end671        if ((c & 0xE0) == 0xC0) {672            // 2-byte character start: 110xxxxx673            // Needs at least 2 bytes674            if (i < 2) return len - i;675        } else if ((c & 0xF0) == 0xE0) {676            // 3-byte character start: 1110xxxx677            // Needs at least 3 bytes678            if (i < 3) return len - i;679        } else if ((c & 0xF8) == 0xF0) {680            // 4-byte character start: 11110xxx681            // Needs at least 4 bytes682            if (i < 4) return len - i;683        }684    }685 686    // If no cut-off multi-byte character is found, return full length687    return len;688}689 690server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {691    // these will be freed upon going out of scope692    mtmd::bitmaps bitmaps;693    std::vector<mtmd_helper::video_ptr> videos;694    for (auto & file : files) {695        auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder);696        if (!out.bitmap) {697            throw std::runtime_error("Failed to load image or audio file");698        }699        bitmaps.entries.emplace_back(out.bitmap);700        if (out.video_ctx) {701            videos.emplace_back(out.video_ctx);702        }703    }704    // process prompt705    std::vector<server_tokens> inputs;706    // multimodal707    mtmd_input_text inp_txt = {708        prompt.data(),709        prompt.size(),710        /* add_special */   true,711        /* parse_special */ true,712    };713    mtmd::input_chunks chunks(mtmd_input_chunks_init());714    auto bitmaps_c_ptr = bitmaps.c_ptr();715    int32_t tokenized = mtmd_tokenize(mctx,716                                      chunks.ptr.get(),717                                      &inp_txt,718                                      bitmaps_c_ptr.data(),719                                      bitmaps_c_ptr.size());720    if (tokenized != 0) {721        throw std::runtime_error("Failed to tokenize prompt");722    }723    auto result = server_tokens(chunks, true);724    return result;725}726 727/**728 * break the input "prompt" object into multiple prompt if needed, then tokenize them729 * use tokenize_input_prompts() if the input could be an array.730 * this supports these cases:731 * - "prompt": "string"732 * - "prompt": [12, 34, 56]733 * - "prompt": [12, 34, "string", 56, 78]734 * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }735 */736static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {737    constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";738    constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";739    const bool has_mtmd = mctx != nullptr;740    if (json_prompt.is_string() || json_is_array_of_mixed_numbers_strings(json_prompt)) {741        // string or mixed742        llama_tokens tmp = tokenize_mixed(vocab, json_prompt, add_special, parse_special);743        return server_tokens(tmp, false);744    } else if (json_is_array_of_numbers(json_prompt)) {745        // array of tokens746        llama_tokens tmp = json_prompt.get<llama_tokens>();747        return server_tokens(tmp, false);748    } else if (json_prompt.contains(JSON_STRING_PROMPT_KEY)) {749        // JSON object with prompt key.750        if (json_prompt.contains(JSON_MTMD_DATA_KEY)) {751            if (!has_mtmd)752                throw std::runtime_error("Multimodal data provided, but model does not support multimodal requests.");753 754            // JSON object with prompt and multimodal key.755            std::vector<raw_buffer> files;756            for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {757                files.push_back(base64_decode(entry));758            }759            return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files);760        } else {761            // Not multimodal, but contains a subobject.762            llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);763            return server_tokens(tmp, false);764        }765   } else {766       throw std::runtime_error("\"prompt\" elements must be a string, a list of tokens, a JSON object containing a prompt string, or a list of mixed strings & tokens.");767   }768}769 770std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {771    std::vector<server_tokens> result;772    if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {773        result.reserve(json_prompt.size());774        for (const auto & p : json_prompt) {775            result.push_back(tokenize_input_subprompt(vocab, mctx, p,add_special, parse_special));776        }777    } else {778        result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special));779    }780    if (result.empty()) {781        throw std::runtime_error("\"prompt\" must not be empty");782    }783    return result;784}785 786//787// OAI utils788//789 790// used by /completions endpoint791json oaicompat_completion_params_parse(const json & body) {792    json llama_params;793 794    if (!body.contains("prompt")) {795        throw std::runtime_error("\"prompt\" is required");796    }797 798    // Handle "stop" field799    if (body.contains("stop") && body.at("stop").is_string()) {800        llama_params["stop"] = json::array({body.at("stop").get<std::string>()});801    } else {802        llama_params["stop"] = json_value(body, "stop", json::array());803    }804 805    // Handle "echo" field806    if (json_value(body, "echo", false)) {807        throw std::runtime_error("Only no echo is supported");808    }809 810    // Params supported by OAI but unsupported by llama.cpp811    static const std::vector<std::string> unsupported_params { "best_of" };812    for (const auto & param : unsupported_params) {813        if (body.contains(param)) {814            throw std::runtime_error("Unsupported param: " + param);815        }816    }817 818    // Copy remaining properties to llama_params819    for (const auto & item : body.items()) {820        // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"821        if (!llama_params.contains(item.key()) || item.key() == "n_predict") {822            llama_params[item.key()] = item.value();823        }824    }825 826    return llama_params;827}828 829// url can be830// - http(s):// for remote files831// - file:// for local files (only allowed if media_path is set)832// - data: for base64 encoded data with uri scheme (e.g. data:image/png;base64,...)833// - raw base64 encoded data834static void handle_media(835        std::vector<raw_buffer> & out_files,836        const std::string & url,837        const std::string & media_path,838        bool accept_base64_uri) {839    if (!media_path.empty()) {840        // should already be enforced by arg.cpp, but checking just in case841        GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);842    }843 844    if (string_starts_with(url, "http")) {845        // download remote image846        // TODO @ngxson : maybe make these params configurable847        common_remote_params params;848        params.max_size = 1024 * 1024 * 10; // 10MB849        params.timeout  = 10; // seconds850        SRV_INF("downloading image from '%s'\n", url.c_str());851        auto res = common_remote_get_content(url, params);852        if (200 <= res.first && res.first < 300) {853            SRV_INF("downloaded %zu bytes\n", res.second.size());854            raw_buffer data;855            data.insert(data.end(), res.second.begin(), res.second.end());856            out_files.push_back(data);857        } else {858            throw std::runtime_error("Failed to download image");859        }860 861    } else if (string_starts_with(url, "file://")) {862        if (media_path.empty()) {863            throw std::invalid_argument("file:// URLs are not allowed unless --media-path is specified");864        }865        // load local image file866        std::string file_path = url.substr(7); // remove "file://"867        raw_buffer data;868        if (!fs_validate_filename(file_path, true)) {869            throw std::invalid_argument("file path is not allowed: " + file_path);870        }871        SRV_INF("loading image from local file '%s'\n", (media_path + file_path).c_str());872        std::ifstream file(media_path + file_path, std::ios::binary);873        if (!file) {874            throw std::invalid_argument("file does not exist or cannot be opened: " + file_path);875        }876        data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());877        out_files.push_back(data);878 879    } else if (accept_base64_uri && string_starts_with(url, "data:")) {880        // try to decode base64 image881        std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');882        if (parts.size() != 2) {883            throw std::runtime_error("Invalid uri-encoded base64 value");884        } else if (!string_starts_with(parts[0], "data:image/")) {885            throw std::runtime_error("Invalid uri format: " + parts[0]);886        } else if (!string_ends_with(parts[0], "base64")) {887            throw std::runtime_error("uri must be base64 encoded");888        } else {889            auto base64_data = parts[1];890            auto decoded_data = base64_decode(base64_data);891            out_files.push_back(decoded_data);892        }893 894    } else {895        // try as raw base64 string896        auto decoded_data = base64_decode(url);897        if (decoded_data.empty()) {898            throw std::runtime_error("Invalid base64 value");899        }900        out_files.push_back(decoded_data);901    }902}903 904// used by /chat/completions endpoint905json oaicompat_chat_params_parse(906    json & body, /* openai api json semantics */907    const server_chat_params & opt,908    std::vector<raw_buffer> & out_files)909{910    json llama_params;911 912    auto tools = json_value(body, "tools", json());913    auto has_tools = tools.is_array() && !tools.empty();914    auto stream = json_value(body, "stream", false);915    auto tool_choice = json_value(body, "tool_choice", std::string("auto"));916 917    if (!opt.use_jinja) {918        if (has_tools) {919            throw std::runtime_error("tools param requires --jinja flag");920        }921        if (tool_choice != "auto") {922            throw std::runtime_error("tool_choice param requires --jinja flag");923        }924    }925 926    // Handle "stop" field927    if (body.contains("stop") && body.at("stop").is_string()) {928        llama_params["stop"] = json::array({body.at("stop").get<std::string>()});929    } else {930        llama_params["stop"] = json_value(body, "stop", json::array());931    }932 933    auto json_schema = json_value(body, "json_schema", json());934    auto grammar = json_value(body, "grammar", std::string());935    if (!json_schema.is_null() && !grammar.empty()) {936        throw std::runtime_error("Cannot use both json_schema and grammar");937    }938 939    // Handle "response_format" field940    if (body.contains("response_format")) {941        json response_format      = json_value(body, "response_format", json::object());942        std::string response_type = json_value(response_format, "type", std::string());943        if (response_type == "json_object") {944            if (response_format.contains("schema") || json_schema.empty()) {945                json_schema = json_value(response_format, "schema", json::object());946            }947        } else if (response_type == "json_schema") {948            auto schema_wrapper = json_value(response_format, "json_schema", json::object());949            json_schema = json_value(schema_wrapper, "schema", json::object());950        } else if (!response_type.empty() && response_type != "text") {951            throw std::invalid_argument("response_format type must be one of \"text\" or \"json_object\", but got: " + response_type);952        }953    }954 955    // get input files956    if (!body.contains("messages")) {957        throw std::invalid_argument("'messages' is required");958    }959    json & messages = body.at("messages");960    if (!messages.is_array()) {961        throw std::invalid_argument("Expected 'messages' to be an array");962    }963    for (auto & msg : messages) {964        std::string role = json_value(msg, "role", std::string());965        if (role != "assistant" && !msg.contains("content")) {966            throw std::invalid_argument("All non-assistant messages must contain 'content'");967        }968        if (role == "assistant") {969            if (!msg.contains("content") && !msg.contains("tool_calls")) {970                throw std::invalid_argument("Assistant message must contain either 'content' or 'tool_calls'!");971            }972            if (!msg.contains("content")) {973                continue; // avoid errors with no content974            }975        }976        json & content = msg.at("content");977        if (content.is_string() || content.is_null()) {978            continue;979        }980 981        if (!content.is_array()) {982            throw std::invalid_argument("Expected 'content' to be a string or an array");983        }984 985        for (auto & p : content) {986            std::string type = json_value(p, "type", std::string());987            if (type == "image_url") {988                if (!opt.allow_image) {989                    throw std::runtime_error("image input is not supported - hint: if this is unexpected, you may need to provide the mmproj");990                }991 992                json image_url = json_value(p, "image_url", json::object());993                std::string url = json_value(image_url, "url", std::string());994                handle_media(out_files, url, opt.media_path, true);995 996                p["type"] = "media_marker";997                p["text"] = get_media_marker();998                p.erase("image_url");999 1000            } else if (type == "input_audio") {1001                if (!opt.allow_audio) {1002                    throw std::runtime_error("audio input is not supported - hint: if this is unexpected, you may need to provide the mmproj");1003                }1004 1005                // note: don't need to validate "format", it's redundant1006                json input_audio = json_value(p, "input_audio", json::object());1007                std::string url  = json_value(input_audio, "data",1008                                        json_value(input_audio, "url", std::string()));1009                handle_media(out_files, url, opt.media_path, false);1010 1011                p["type"] = "media_marker";1012                p["text"] = get_media_marker();1013                p.erase("input_audio");1014 1015            } else if (type == "input_video") {1016                if (!opt.allow_video) {1017                    throw std::runtime_error("video input is not supported - hint: if this is unexpected, you may need to provide the mmproj");1018                }1019 1020                json input_video = json_value(p, "input_video", json::object());1021                std::string url  = json_value(input_video, "data",1022                                        json_value(input_video, "url", std::string()));1023                handle_media(out_files, url, opt.media_path, false);1024 1025                p["type"] = "media_marker";1026                p["text"] = get_media_marker();1027                p.erase("input_video");1028 1029            } else if (type != "text") {1030                throw std::invalid_argument("unsupported content[].type");1031            }1032        }1033    }1034 1035    auto caps = common_chat_templates_get_caps(opt.tmpls.get());1036 1037    common_chat_templates_inputs inputs;1038    inputs.messages               = common_chat_msgs_parse_oaicompat(messages);1039    inputs.tools                  = common_chat_tools_parse_oaicompat(tools);1040    inputs.tool_choice            = common_chat_tool_choice_parse_oaicompat(tool_choice);1041    inputs.json_schema            = json_schema.is_null() ? "" : json_schema.dump();1042    inputs.grammar                = grammar;1043    inputs.use_jinja              = opt.use_jinja;1044    inputs.parallel_tool_calls    = json_value(body, "parallel_tool_calls", caps["supports_parallel_tool_calls"]);1045    inputs.add_generation_prompt  = json_value(body, "add_generation_prompt", true);1046    inputs.continue_final_message = body.contains("continue_final_message") ?1047        common_chat_continuation_parse(body.at("continue_final_message")) :1048        COMMON_CHAT_CONTINUATION_NONE;1049    if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_NONE && opt.prefill_assistant1050        && !inputs.messages.empty() && inputs.messages.back().role == "assistant") {1051        if (inputs.messages.size() >= 2 && inputs.messages[inputs.messages.size() - 2].role == "assistant") {1052            throw std::invalid_argument("Cannot have 2 or more assistant messages at the end of the list.");1053        }1054        inputs.continue_final_message = COMMON_CHAT_CONTINUATION_AUTO;1055        inputs.add_generation_prompt  = false;1056    }1057    if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && inputs.add_generation_prompt) {1058        throw std::invalid_argument("Cannot set both add_generation_prompt and continue_final_message to true.");1059    }1060    inputs.reasoning_format = opt.reasoning_format;1061    if (body.contains("reasoning_format")) {1062        inputs.reasoning_format = common_reasoning_format_from_name(body.at("reasoning_format").get<std::string>());1063    }1064    inputs.enable_thinking = opt.enable_thinking;1065    if (!inputs.tools.empty() && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {1066        if (body.contains("grammar")) {1067            throw std::invalid_argument("Cannot use custom grammar constraints with tools.");1068        }1069        llama_params["parse_tool_calls"] = true;1070    }1071 1072    // merge the template args provided from command line with the args provided in the user request1073    auto chat_template_kwargs_object = json_value(body, "chat_template_kwargs", json::object());1074    inputs.chat_template_kwargs = opt.chat_template_kwargs;1075    for (const auto & item : chat_template_kwargs_object.items()) {1076        inputs.chat_template_kwargs[item.key()] = item.value().dump();1077    }1078 1079    // parse the "enable_thinking" kwarg to override the default value1080    auto enable_thinking_kwarg = json_value(inputs.chat_template_kwargs, "enable_thinking", std::string(""));1081    if (enable_thinking_kwarg == "true") {1082        inputs.enable_thinking = true;1083    } else if (enable_thinking_kwarg == "false") {1084        inputs.enable_thinking = false;1085    } else if (!enable_thinking_kwarg.empty() && enable_thinking_kwarg[0] == '"') {1086        throw std::invalid_argument("invalid type for \"enable_thinking\" (expected boolean, got string)");1087    }1088 1089    // Parse also the OAI "reasoning_effort": "none" specific value1090    if (body.contains("reasoning_effort")) {1091        auto reasoning_effort = json_value(body, "reasoning_effort", std::string(""));1092        if (reasoning_effort == "none") {1093            inputs.enable_thinking = false;1094        } // other reasoning_effort values are model-specific and not yet handled1095    }1096 1097    inputs.force_pure_content = opt.force_pure_content;1098 1099    // Apply chat template to the list of messages1100    auto chat_params = common_chat_templates_apply(opt.tmpls.get(), inputs);1101 1102    llama_params["chat_format"] = static_cast<int>(chat_params.format);1103    llama_params["prompt"]      = chat_params.prompt;1104    if (!chat_params.grammar.empty()) {1105        llama_params["grammar"]      = chat_params.grammar;1106        llama_params["grammar_type"] = std::string("tool_calls");1107    }1108    llama_params["grammar_lazy"] = chat_params.grammar_lazy;1109    auto grammar_triggers        = json::array();1110    for (const auto & trigger : chat_params.grammar_triggers) {1111        server_grammar_trigger ct(trigger);1112        grammar_triggers.push_back(ct.to_json());1113    }1114    llama_params["grammar_triggers"]  = grammar_triggers;1115    llama_params["preserved_tokens"]  = chat_params.preserved_tokens;1116    llama_params["generation_prompt"] = chat_params.generation_prompt;1117    for (const auto & stop : chat_params.additional_stops) {1118        llama_params["stop"].push_back(stop);1119    }1120    if (!chat_params.parser.empty()) {1121        llama_params["chat_parser"] = chat_params.parser;1122    }1123 1124    llama_params["message_delimiters"] = chat_params.message_delimiters.to_json();1125 1126    // Reasoning budget: pass parameters through to sampling layer1127    {1128        int reasoning_budget = json_value(body, "reasoning_budget_tokens",1129                               json_value(body, "thinking_budget_tokens", -1));1130        if (reasoning_budget == -1) {1131            reasoning_budget = opt.reasoning_budget;1132        }1133 1134        if (!chat_params.thinking_end_tags.empty()) {1135            llama_params["reasoning_budget_tokens"] = reasoning_budget;1136            llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;1137            llama_params["reasoning_budget_end_tags"] = chat_params.thinking_end_tags;1138            llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message);1139            llama_params["reasoning_control"] = json_value(body, "reasoning_control", false);1140        }1141    }1142 1143    // Handle "logprobs" field1144    // TODO: The response format of this option is not yet OAI-compatible, but seems like no one really using it; We may need to fix it in the future1145    if (json_value(body, "logprobs", false)) {1146        if (has_tools && stream) {1147            throw std::invalid_argument("logprobs is not supported with tools + stream");1148        }1149        llama_params["n_probs"] = json_value(body, "top_logprobs", 20);1150    } else if (body.contains("top_logprobs") && !body.at("top_logprobs").is_null()) {1151        throw std::invalid_argument("top_logprobs requires logprobs to be set to true");1152    }1153 1154    // Copy remaining properties to llama_params1155    // This allows user to use llama.cpp-specific params like "mirostat", ... via OAI endpoint.1156    // See "launch_slot_with_task()" for a complete list of params supported by llama.cpp1157    for (const auto & item : body.items()) {1158        // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"1159        if (!llama_params.contains(item.key()) || item.key() == "n_predict") {1160            llama_params[item.key()] = item.value();1161        }1162    }1163 1164    return llama_params;1165}1166 1167json format_embeddings_response_oaicompat(1168        const json & request,1169        const std::string & model_name,1170        const json & embeddings,1171        bool use_base64) {1172    json data = json::array();1173    int32_t n_tokens = 0;1174    int i = 0;1175    for (const auto & elem : embeddings) {1176        json embedding_obj;1177 1178        if (use_base64) {1179            const auto& vec = json_value(elem, "embedding", json::array()).get<std::vector<float>>();1180            const char* data_ptr = reinterpret_cast<const char*>(vec.data());1181            size_t data_size = vec.size() * sizeof(float);1182            embedding_obj = {1183                {"embedding", base64::encode(data_ptr, data_size)},1184                {"index", i++},1185                {"object", "embedding"},1186                {"encoding_format", "base64"}1187            };1188        } else {1189            embedding_obj = {1190                {"embedding", json_value(elem, "embedding", json::array())},1191                {"index", i++},1192                {"object", "embedding"}1193            };1194        }1195        data.push_back(embedding_obj);1196 1197        n_tokens += json_value(elem, "tokens_evaluated", 0);1198    }1199 1200    json res = json {

Showing the first 1,200 of 1596 lines. Download the file for the rest.