KBaba7/llama.cpp
0
1#pragma once2 3#include "common.h"4#include "log.h"5#include "llama.h"6#include "common/base64.hpp"7 8// increase max payload length to allow use of larger context size9#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 104857610#include "httplib.h"11 12// Change JSON_ASSERT from assert() to GGML_ASSERT:13#define JSON_ASSERT GGML_ASSERT14#include "json.hpp"15#include "minja.hpp"16#include "chat.hpp"17#include "chat-template.hpp"18 19#include <random>20#include <sstream>21#include <string>22#include <vector>23#include <memory>24 25#define DEFAULT_OAICOMPAT_MODEL "gpt-3.5-turbo"26 27using json = nlohmann::ordered_json;28 29#define SLT_INF(slot, fmt, ...) LOG_INF("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__)30#define SLT_WRN(slot, fmt, ...) LOG_WRN("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__)31#define SLT_ERR(slot, fmt, ...) LOG_ERR("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__)32#define SLT_DBG(slot, fmt, ...) LOG_DBG("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, (slot).id_task, __VA_ARGS__)33 34#define SRV_INF(fmt, ...) LOG_INF("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)35#define SRV_WRN(fmt, ...) LOG_WRN("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)36#define SRV_ERR(fmt, ...) LOG_ERR("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)37#define SRV_DBG(fmt, ...) LOG_DBG("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)38 39#define QUE_INF(fmt, ...) LOG_INF("que %12.*s: " fmt, 12, __func__, __VA_ARGS__)40#define QUE_WRN(fmt, ...) LOG_WRN("que %12.*s: " fmt, 12, __func__, __VA_ARGS__)41#define QUE_ERR(fmt, ...) LOG_ERR("que %12.*s: " fmt, 12, __func__, __VA_ARGS__)42#define QUE_DBG(fmt, ...) LOG_DBG("que %12.*s: " fmt, 12, __func__, __VA_ARGS__)43 44template <typename T>45static T json_value(const json & body, const std::string & key, const T & default_value) {46 // Fallback null to default value47 if (body.contains(key) && !body.at(key).is_null()) {48 try {49 return body.at(key);50 } catch (NLOHMANN_JSON_NAMESPACE::detail::type_error const &) {51 LOG_WRN("Wrong type supplied for parameter '%s'. Expected '%s', using default value\n", key.c_str(), json(default_value).type_name());52 return default_value;53 }54 } else {55 return default_value;56 }57}58 59const static std::string build_info("b" + std::to_string(LLAMA_BUILD_NUMBER) + "-" + LLAMA_COMMIT);60 61//62// tokenizer and input processing utils63//64 65static bool json_is_array_of_numbers(const json & data) {66 if (data.is_array()) {67 for (const auto & e : data) {68 if (!e.is_number_integer()) {69 return false;70 }71 }72 return true;73 }74 return false;75}76 77// is array having BOTH numbers & strings?78static bool json_is_array_of_mixed_numbers_strings(const json & data) {79 bool seen_string = false;80 bool seen_number = false;81 if (data.is_array()) {82 for (const auto & e : data) {83 seen_string |= e.is_string();84 seen_number |= e.is_number_integer();85 if (seen_number && seen_string) {86 return true;87 }88 }89 }90 return false;91}92 93// get value by path(key1 / key2)94static json json_get_nested_values(const std::vector<std::string> & paths, const json & js) {95 json result = json::object();96 97 for (const std::string & path : paths) {98 json current = js;99 const auto keys = string_split<std::string>(path, /*separator*/ '/');100 bool valid_path = true;101 for (const std::string & k : keys) {102 if (valid_path && current.is_object() && current.contains(k)) {103 current = current[k];104 } else {105 valid_path = false;106 }107 }108 if (valid_path) {109 result[path] = current;110 }111 }112 return result;113}114 115/**116 * this handles 2 cases:117 * - only string, example: "string"118 * - mixed string and tokens, example: [12, 34, "string", 56, 78]119 */120static llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special) {121 // If `add_bos` is true, we only add BOS, when json_prompt is a string,122 // or the first element of the json_prompt array is a string.123 llama_tokens prompt_tokens;124 125 if (json_prompt.is_array()) {126 bool first = true;127 for (const auto & p : json_prompt) {128 if (p.is_string()) {129 auto s = p.template get<std::string>();130 131 llama_tokens p;132 if (first) {133 p = common_tokenize(vocab, s, add_special, parse_special);134 first = false;135 } else {136 p = common_tokenize(vocab, s, false, parse_special);137 }138 139 prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());140 } else {141 if (first) {142 first = false;143 }144 145 prompt_tokens.push_back(p.template get<llama_token>());146 }147 }148 } else {149 auto s = json_prompt.template get<std::string>();150 prompt_tokens = common_tokenize(vocab, s, add_special, parse_special);151 }152 153 return prompt_tokens;154}155 156/**157 * break the input "prompt" object into multiple prompt if needed, then tokenize them158 * this supports these cases:159 * - "prompt": "string"160 * - "prompt": [12, 34, 56]161 * - "prompt": [12, 34, "string", 56, 78]162 * and multiple prompts (multi-tasks):163 * - "prompt": ["string1", "string2"]164 * - "prompt": ["string1", [12, 34, 56]]165 * - "prompt": [[12, 34, 56], [78, 90, 12]]166 * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56]]167 */168static std::vector<llama_tokens> tokenize_input_prompts(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special) {169 std::vector<llama_tokens> result;170 if (json_prompt.is_string() || json_is_array_of_mixed_numbers_strings(json_prompt)) {171 // string or mixed172 result.push_back(tokenize_mixed(vocab, json_prompt, add_special, parse_special));173 } else if (json_is_array_of_numbers(json_prompt)) {174 // array of tokens175 result.push_back(json_prompt.get<llama_tokens>());176 } else if (json_prompt.is_array()) {177 // array of prompts178 result.reserve(json_prompt.size());179 for (const auto & p : json_prompt) {180 if (p.is_string() || json_is_array_of_mixed_numbers_strings(p)) {181 result.push_back(tokenize_mixed(vocab, p, add_special, parse_special));182 } else if (json_is_array_of_numbers(p)) {183 // array of tokens184 result.push_back(p.get<llama_tokens>());185 } else {186 throw std::runtime_error("element of \"prompt\" must be a string, an list of tokens, or a list of mixed strings & tokens");187 }188 }189 } else {190 throw std::runtime_error("\"prompt\" must be a string, an list of tokens, a list of mixed strings & tokens, or a list of prompts");191 }192 if (result.empty()) {193 throw std::runtime_error("\"prompt\" must not be empty");194 }195 return result;196}197 198// return the last index of character that can form a valid string199// if the last character is potentially cut in half, return the index before the cut200// if validate_utf8(text) == text.size(), then the whole text is valid utf8201static size_t validate_utf8(const std::string& text) {202 size_t len = text.size();203 if (len == 0) return 0;204 205 // Check the last few bytes to see if a multi-byte character is cut off206 for (size_t i = 1; i <= 4 && i <= len; ++i) {207 unsigned char c = text[len - i];208 // Check for start of a multi-byte sequence from the end209 if ((c & 0xE0) == 0xC0) {210 // 2-byte character start: 110xxxxx211 // Needs at least 2 bytes212 if (i < 2) return len - i;213 } else if ((c & 0xF0) == 0xE0) {214 // 3-byte character start: 1110xxxx215 // Needs at least 3 bytes216 if (i < 3) return len - i;217 } else if ((c & 0xF8) == 0xF0) {218 // 4-byte character start: 11110xxx219 // Needs at least 4 bytes220 if (i < 4) return len - i;221 }222 }223 224 // If no cut-off multi-byte character is found, return full length225 return len;226}227 228//229// template utils230//231 232// format rerank task: [BOS]query[EOS][SEP]doc[EOS]233static llama_tokens format_rerank(const struct llama_vocab * vocab, const llama_tokens & query, const llama_tokens & doc) {234 llama_tokens result;235 236 result.reserve(doc.size() + query.size() + 4);237 result.push_back(llama_vocab_bos(vocab));238 result.insert(result.end(), query.begin(), query.end());239 result.push_back(llama_vocab_eos(vocab));240 result.push_back(llama_vocab_sep(vocab));241 result.insert(result.end(), doc.begin(), doc.end());242 result.push_back(llama_vocab_eos(vocab));243 244 return result;245}246 247// format infill task248static llama_tokens format_infill(249 const llama_vocab * vocab,250 const json & input_prefix,251 const json & input_suffix,252 const json & input_extra,253 const int n_batch,254 const int n_predict,255 const int n_ctx,256 const bool spm_infill,257 const llama_tokens & tokens_prompt258 ) {259 // TODO: optimize this block by reducing memory allocations and movement260 261 // use FIM repo-level pattern:262 // ref: https://arxiv.org/pdf/2409.12186263 //264 // [FIM_REP]myproject265 // [FIM_SEP]filename0266 // extra chunk 0267 // [FIM_SEP]filename1268 // extra chunk 1269 // ...270 // [FIM_SEP]filename271 // [FIM_PRE]prefix[FIM_SUF]suffix[FIM_MID]prompt272 //273 llama_tokens extra_tokens;274 extra_tokens.reserve(n_ctx);275 276 auto tokens_prefix = tokenize_mixed(vocab, input_prefix, false, false);277 auto tokens_suffix = tokenize_mixed(vocab, input_suffix, false, false);278 279 if (llama_vocab_fim_rep(vocab) != LLAMA_TOKEN_NULL) {280 // TODO: make project name an input281 static const auto k_fim_repo = common_tokenize(vocab, "myproject\n", false, false);282 283 extra_tokens.push_back(llama_vocab_fim_rep(vocab));284 extra_tokens.insert(extra_tokens.end(), k_fim_repo.begin(), k_fim_repo.end());285 }286 for (const auto & chunk : input_extra) {287 // { "text": string, "filename": string }288 const std::string text = json_value(chunk, "text", std::string());289 const std::string filename = json_value(chunk, "filename", std::string("tmp"));290 291 if (llama_vocab_fim_sep(vocab) != LLAMA_TOKEN_NULL) {292 const auto k_fim_file = common_tokenize(vocab, filename + "\n", false, false);293 294 extra_tokens.insert(extra_tokens.end(), llama_vocab_fim_sep(vocab));295 extra_tokens.insert(extra_tokens.end(), k_fim_file.begin(), k_fim_file.end());296 } else {297 // chunk separator in binary form to avoid confusing the AI298 static const char k_chunk_prefix_str[] = {0x0a, 0x0a, 0x2d, 0x2d, 0x2d, 0x20, 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65, 0x74, 0x20, 0x2d, 0x2d, 0x2d, 0x0a, 0x0a, 0x00};299 static const auto k_chunk_prefix_tokens = common_tokenize(vocab, k_chunk_prefix_str, false, false);300 301 extra_tokens.insert(extra_tokens.end(), k_chunk_prefix_tokens.begin(), k_chunk_prefix_tokens.end());302 }303 304 const auto chunk_tokens = common_tokenize(vocab, text, false, false);305 extra_tokens.insert(extra_tokens.end(), chunk_tokens.begin(), chunk_tokens.end());306 }307 308 if (llama_vocab_fim_sep(vocab) != LLAMA_TOKEN_NULL) {309 // TODO: current filename310 static const auto k_fim_file = common_tokenize(vocab, "filename\n", false, false);311 312 extra_tokens.insert(extra_tokens.end(), llama_vocab_fim_sep(vocab));313 extra_tokens.insert(extra_tokens.end(), k_fim_file.begin(), k_fim_file.end());314 }315 316 // for now pick FIM context to fit in a batch (ratio prefix:suffix = 3:1, TODO: configurable?)317 const int n_prefix_take = std::min<int>(tokens_prefix.size(), 3*(n_batch/4));318 const int n_suffix_take = std::min<int>(tokens_suffix.size(), std::max<int>(0, (n_batch/4) - (2 + tokens_prompt.size())));319 320 SRV_DBG("n_prefix_take = %d, n_suffix_take = %d, total = %d\n", n_prefix_take, n_suffix_take, (n_prefix_take + n_suffix_take));321 322 // fill the rest of the context with extra chunks323 const int n_extra_take = std::min<int>(std::max<int>(0, n_ctx - (n_batch) - 2*n_predict), extra_tokens.size());324 325 tokens_prefix.erase(tokens_prefix.begin(), tokens_prefix.begin() + tokens_prefix.size() - n_prefix_take);326 tokens_suffix.resize(n_suffix_take);327 328 tokens_prefix.insert(tokens_prefix.begin(), llama_vocab_fim_pre(vocab));329 tokens_prefix.insert(tokens_prefix.end(), tokens_prompt.begin(), tokens_prompt.end());330 tokens_suffix.insert(tokens_suffix.begin(), llama_vocab_fim_suf(vocab));331 332 auto embd_inp = spm_infill ? tokens_suffix : tokens_prefix;333 auto embd_end = spm_infill ? tokens_prefix : tokens_suffix;334 335 if (llama_vocab_get_add_bos(vocab)) {336 embd_inp.insert(embd_inp.begin(), llama_vocab_bos(vocab));337 }338 339 SRV_DBG("extra: n_ctx = %d, n_extra_take = %d, n_extra = %d\n", n_ctx, n_extra_take, (int) extra_tokens.size());340 341 // put the extra context before the FIM prefix342 embd_inp.insert(embd_inp.begin(), extra_tokens.end() - n_extra_take, extra_tokens.end());343 344 embd_inp.insert(embd_inp.end(), embd_end.begin(), embd_end.end());345 embd_inp.push_back(llama_vocab_fim_mid(vocab));346 347 return embd_inp;348}349 350// Format given chat. If tmpl is empty, we take the template from model metadata351inline std::string format_chat(const common_chat_template & tmpl, const std::vector<json> & messages) {352 std::vector<common_chat_msg> chat;353 354 for (size_t i = 0; i < messages.size(); ++i) {355 const auto & curr_msg = messages[i];356 357 std::string role = json_value(curr_msg, "role", std::string(""));358 359 std::string content;360 if (curr_msg.contains("content")) {361 if (curr_msg["content"].is_string()) {362 content = curr_msg["content"].get<std::string>();363 } else if (curr_msg["content"].is_array()) {364 for (const auto & part : curr_msg["content"]) {365 if (part.contains("text")) {366 content += "\n" + part["text"].get<std::string>();367 }368 }369 } else {370 throw std::runtime_error("Invalid 'content' type (ref: https://github.com/ggerganov/llama.cpp/issues/8367)");371 }372 } else {373 throw std::runtime_error("Missing 'content' (ref: https://github.com/ggerganov/llama.cpp/issues/8367)");374 }375 376 chat.push_back({role, content, /* tool_calls= */ {}});377 }378 379 const auto formatted_chat = common_chat_apply_template(tmpl, chat, true, /* use_jinja= */ false);380 LOG_DBG("formatted_chat: '%s'\n", formatted_chat.c_str());381 382 return formatted_chat;383}384 385//386// base64 utils (TODO: move to common in the future)387//388 389static const std::string base64_chars =390 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"391 "abcdefghijklmnopqrstuvwxyz"392 "0123456789+/";393 394static inline bool is_base64(uint8_t c) {395 return (isalnum(c) || (c == '+') || (c == '/'));396}397 398static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string) {399 int i = 0;400 int j = 0;401 int in_ = 0;402 403 int in_len = encoded_string.size();404 405 uint8_t char_array_4[4];406 uint8_t char_array_3[3];407 408 std::vector<uint8_t> ret;409 410 while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {411 char_array_4[i++] = encoded_string[in_]; in_++;412 if (i == 4) {413 for (i = 0; i < 4; i++) {414 char_array_4[i] = base64_chars.find(char_array_4[i]);415 }416 417 char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);418 char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);419 char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];420 421 for (i = 0; (i < 3); i++) {422 ret.push_back(char_array_3[i]);423 }424 425 i = 0;426 }427 }428 429 if (i) {430 for (j = i; j < 4; j++) {431 char_array_4[j] = 0;432 }433 434 for (j = 0; j < 4; j++) {435 char_array_4[j] = base64_chars.find(char_array_4[j]);436 }437 438 char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);439 char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);440 char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];441 442 for (j = 0; j < i - 1; j++) {443 ret.push_back(char_array_3[j]);444 }445 }446 447 return ret;448}449 450//451// random string / id452//453 454static std::string random_string() {455 static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");456 457 std::random_device rd;458 std::mt19937 generator(rd());459 460 std::string result(32, ' ');461 462 for (int i = 0; i < 32; ++i) {463 result[i] = str[generator() % str.size()];464 }465 466 return result;467}468 469static std::string gen_chatcmplid() {470 return "chatcmpl-" + random_string();471}472 473//474// other common utils475//476 477static bool ends_with(const std::string & str, const std::string & suffix) {478 return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);479}480 481static size_t find_partial_stop_string(const std::string &stop, const std::string &text) {482 if (!text.empty() && !stop.empty()) {483 const char text_last_char = text.back();484 for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--) {485 if (stop[char_index] == text_last_char) {486 const std::string current_partial = stop.substr(0, char_index + 1);487 if (ends_with(text, current_partial)) {488 return text.size() - char_index - 1;489 }490 }491 }492 }493 494 return std::string::npos;495}496 497// TODO: reuse llama_detokenize498template <class Iter>499static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) {500 std::string ret;501 for (; begin != end; ++begin) {502 ret += common_token_to_piece(ctx, *begin);503 }504 505 return ret;506}507 508// format incomplete utf-8 multibyte character for output509static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {510 std::string out = token == LLAMA_TOKEN_NULL ? "" : common_token_to_piece(ctx, token);511 512 // if the size is 1 and first bit is 1, meaning it's a partial character513 // (size > 1 meaning it's already a known token)514 if (out.size() == 1 && (out[0] & 0x80) == 0x80) {515 std::stringstream ss;516 ss << std::hex << (out[0] & 0xff);517 std::string res(ss.str());518 out = "byte: \\x" + res;519 }520 521 return out;522}523 524static bool server_sent_event(httplib::DataSink & sink, const char * event, const json & data) {525 const std::string str =526 std::string(event) + ": " +527 data.dump(-1, ' ', false, json::error_handler_t::replace) +528 "\n\n"; // required by RFC 8895 - A message is terminated by a blank line (two line terminators in a row).529 530 LOG_DBG("data stream, to_send: %s", str.c_str());531 532 return sink.write(str.c_str(), str.size());533}534 535//536// OAI utils537//538 539static json oaicompat_completion_params_parse(const json & body) {540 json llama_params;541 542 if (!body.contains("prompt")) {543 throw std::runtime_error("\"prompt\" is required");544 }545 546 // Handle "stop" field547 if (body.contains("stop") && body.at("stop").is_string()) {548 llama_params["stop"] = json::array({body.at("stop").get<std::string>()});549 } else {550 llama_params["stop"] = json_value(body, "stop", json::array());551 }552 553 // Handle "n" field554 int n_choices = json_value(body, "n", 1);555 if (n_choices != 1) {556 throw std::runtime_error("Only one completion choice is allowed");557 }558 559 // Params supported by OAI but unsupported by llama.cpp560 static const std::vector<std::string> unsupported_params { "best_of", "echo", "suffix" };561 for (const auto & param : unsupported_params) {562 if (body.contains(param)) {563 throw std::runtime_error("Unsupported param: " + param);564 }565 }566 567 // Copy remaining properties to llama_params568 for (const auto & item : body.items()) {569 // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"570 if (!llama_params.contains(item.key()) || item.key() == "n_predict") {571 llama_params[item.key()] = item.value();572 }573 }574 575 return llama_params;576}577 578static json oaicompat_completion_params_parse(579 const json & body, /* openai api json semantics */580 bool use_jinja,581 const common_chat_templates & chat_templates)582{583 json llama_params;584 const auto & tmpl = body.contains("tools") && chat_templates.template_tool_use585 ? *chat_templates.template_tool_use586 : *chat_templates.template_default;587 588 auto tools = json_value(body, "tools", json());589 auto stream = json_value(body, "stream", false);590 591 if (tools.is_array() && !tools.empty()) {592 if (stream) {593 throw std::runtime_error("Cannot use tools with stream");594 }595 if (!use_jinja) {596 throw std::runtime_error("tools param requires --jinja flag");597 }598 }599 if (!use_jinja) {600 if (body.contains("tool_choice") && !body.at("tool_choice").is_null()) {601 throw std::runtime_error("Unsupported param: tool_choice");602 }603 }604 605 // Handle "stop" field606 if (body.contains("stop") && body.at("stop").is_string()) {607 llama_params["stop"] = json::array({body.at("stop").get<std::string>()});608 } else {609 llama_params["stop"] = json_value(body, "stop", json::array());610 }611 612 // Handle "response_format" field613 if (body.contains("response_format")) {614 json response_format = json_value(body, "response_format", json::object());615 std::string response_type = json_value(response_format, "type", std::string());616 if (response_type == "json_object") {617 llama_params["json_schema"] = json_value(response_format, "schema", json::object());618 } else if (response_type == "json_schema") {619 json json_schema = json_value(response_format, "json_schema", json::object());620 llama_params["json_schema"] = json_value(json_schema, "schema", json::object());621 } else if (!response_type.empty() && response_type != "text") {622 throw std::runtime_error("response_format type must be one of \"text\" or \"json_object\", but got: " + response_type);623 }624 }625 626 // Apply chat template to the list of messages627 if (use_jinja) {628 auto tool_choice = json_value(body, "tool_choice", std::string("auto"));629 if (tool_choice != "none" && tool_choice != "auto" && tool_choice != "required") {630 throw std::runtime_error("Invalid tool_choice: " + tool_choice);631 }632 if (tool_choice != "none" && llama_params.contains("grammar")) {633 throw std::runtime_error("Cannot use custom grammar constraints with tools.");634 }635 common_chat_inputs inputs;636 inputs.messages = body.at("messages");637 inputs.tools = tools;638 inputs.tool_choice = tool_choice;639 inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", false);640 if (inputs.parallel_tool_calls && !tmpl.original_caps().supports_parallel_tool_calls) {641 LOG_DBG("Disabling parallel_tool_calls because the template does not support it\n");642 inputs.parallel_tool_calls = false;643 }644 inputs.stream = stream;645 // TODO: support mixing schema w/ tools beyond generic format.646 inputs.json_schema = json_value(llama_params, "json_schema", json());647 auto chat_params = common_chat_params_init(tmpl, inputs);648 649 llama_params["chat_format"] = static_cast<int>(chat_params.format);650 llama_params["prompt"] = chat_params.prompt;651 llama_params["grammar"] = chat_params.grammar;652 llama_params["grammar_lazy"] = chat_params.grammar_lazy;653 auto grammar_triggers = json::array();654 for (const auto & trigger : chat_params.grammar_triggers) {655 grammar_triggers.push_back({656 {"word", trigger.word},657 {"at_start", trigger.at_start},658 });659 }660 llama_params["grammar_triggers"] = grammar_triggers;661 llama_params["preserved_tokens"] = chat_params.preserved_tokens;662 for (const auto & stop : chat_params.additional_stops) {663 llama_params["stop"].push_back(stop);664 }665 } else {666 llama_params["prompt"] = format_chat(tmpl, body.at("messages"));667 }668 669 // Handle "n" field670 int n_choices = json_value(body, "n", 1);671 if (n_choices != 1) {672 throw std::runtime_error("Only one completion choice is allowed");673 }674 675 // Handle "logprobs" field676 // TODO: The response format of this option is not yet OAI-compatible, but seems like no one really using it; We may need to fix it in the future677 if (json_value(body, "logprobs", false)) {678 llama_params["n_probs"] = json_value(body, "top_logprobs", 20);679 } else if (body.contains("top_logprobs") && !body.at("top_logprobs").is_null()) {680 throw std::runtime_error("top_logprobs requires logprobs to be set to true");681 }682 683 // Copy remaining properties to llama_params684 // This allows user to use llama.cpp-specific params like "mirostat", ... via OAI endpoint.685 // See "launch_slot_with_task()" for a complete list of params supported by llama.cpp686 for (const auto & item : body.items()) {687 // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"688 if (!llama_params.contains(item.key()) || item.key() == "n_predict") {689 llama_params[item.key()] = item.value();690 }691 }692 693 return llama_params;694}695 696static json format_embeddings_response_oaicompat(const json & request, const json & embeddings, bool use_base64 = false) {697 json data = json::array();698 int32_t n_tokens = 0;699 int i = 0;700 for (const auto & elem : embeddings) {701 json embedding_obj;702 703 if (use_base64) {704 const auto& vec = json_value(elem, "embedding", json::array()).get<std::vector<float>>();705 const char* data_ptr = reinterpret_cast<const char*>(vec.data());706 size_t data_size = vec.size() * sizeof(float);707 embedding_obj = {708 {"embedding", base64::encode(data_ptr, data_size)},709 {"index", i++},710 {"object", "embedding"},711 {"encoding_format", "base64"}712 };713 } else {714 embedding_obj = {715 {"embedding", json_value(elem, "embedding", json::array())},716 {"index", i++},717 {"object", "embedding"}718 };719 }720 data.push_back(embedding_obj);721 722 n_tokens += json_value(elem, "tokens_evaluated", 0);723 }724 725 json res = json {726 {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},727 {"object", "list"},728 {"usage", json {729 {"prompt_tokens", n_tokens},730 {"total_tokens", n_tokens}731 }},732 {"data", data}733 };734 735 return res;736}737 738static json format_response_rerank(const json & request, const json & ranks) {739 json data = json::array();740 int32_t n_tokens = 0;741 int i = 0;742 for (const auto & rank : ranks) {743 data.push_back(json{744 {"index", i++},745 {"relevance_score", json_value(rank, "score", 0.0)},746 });747 748 n_tokens += json_value(rank, "tokens_evaluated", 0);749 }750 751 json res = json {752 {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},753 {"object", "list"},754 {"usage", json {755 {"prompt_tokens", n_tokens},756 {"total_tokens", n_tokens}757 }},758 {"results", data}759 };760 761 return res;762}763 764static bool is_valid_utf8(const std::string & str) {765 const unsigned char* bytes = reinterpret_cast<const unsigned char*>(str.data());766 const unsigned char* end = bytes + str.length();767 768 while (bytes < end) {769 if (*bytes <= 0x7F) {770 // 1-byte sequence (0xxxxxxx)771 bytes++;772 } else if ((*bytes & 0xE0) == 0xC0) {773 // 2-byte sequence (110xxxxx 10xxxxxx)774 if (end - bytes < 2 || (bytes[1] & 0xC0) != 0x80)775 return false;776 bytes += 2;777 } else if ((*bytes & 0xF0) == 0xE0) {778 // 3-byte sequence (1110xxxx 10xxxxxx 10xxxxxx)779 if (end - bytes < 3 || (bytes[1] & 0xC0) != 0x80 || (bytes[2] & 0xC0) != 0x80)780 return false;781 bytes += 3;782 } else if ((*bytes & 0xF8) == 0xF0) {783 // 4-byte sequence (11110xxx 10xxxxxx 10xxxxxx 10xxxxxx)784 if (end - bytes < 4 || (bytes[1] & 0xC0) != 0x80 ||785 (bytes[2] & 0xC0) != 0x80 || (bytes[3] & 0xC0) != 0x80)786 return false;787 bytes += 4;788 } else {789 // Invalid UTF-8 lead byte790 return false;791 }792 }793 794 return true;795}796 797static json format_tokenizer_response(const json & tokens) {798 return json {799 {"tokens", tokens}800 };801}802 803static json format_detokenized_response(const std::string & content) {804 return json {805 {"content", content}806 };807}808 809static json format_logit_bias(const std::vector<llama_logit_bias> & logit_bias) {810 json data = json::array();811 for (const auto & lb : logit_bias) {812 data.push_back(json{813 {"bias", lb.bias},814 {"token", lb.token},815 });816 }817 return data;818}819 820static std::string safe_json_to_str(const json & data) {821 return data.dump(-1, ' ', false, json::error_handler_t::replace);822}823 824static std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx) {825 std::vector<llama_token_data> cur;826 const auto * logits = llama_get_logits_ith(ctx, idx);827 828 const llama_model * model = llama_get_model(ctx);829 const llama_vocab * vocab = llama_model_get_vocab(model);830 831 const int n_vocab = llama_vocab_n_tokens(vocab);832 833 cur.resize(n_vocab);834 for (llama_token token_id = 0; token_id < n_vocab; token_id++) {835 cur[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};836 }837 838 // sort tokens by logits839 std::sort(cur.begin(), cur.end(), [](const llama_token_data & a, const llama_token_data & b) {840 return a.logit > b.logit;841 });842 843 // apply softmax844 float max_l = cur[0].logit;845 float cum_sum = 0.0f;846 for (size_t i = 0; i < cur.size(); ++i) {847 float p = expf(cur[i].logit - max_l);848 cur[i].p = p;849 cum_sum += p;850 }851 for (size_t i = 0; i < cur.size(); ++i) {852 cur[i].p /= cum_sum;853 }854 855 return cur;856}857 858static bool are_lora_equal(859 const std::vector<common_adapter_lora_info> & l1,860 const std::vector<common_adapter_lora_info> & l2) {861 if (l1.size() != l2.size()) {862 return false;863 }864 for (size_t i = 0; i < l1.size(); ++i) {865 // we don't check lora.path to reduce the time complexity866 if (l1[i].scale != l2[i].scale || l1[i].ptr != l2[i].ptr) {867 return false;868 }869 }870 return true;871}872 873// parse lora config from JSON request, returned a copy of lora_base with updated scale874static std::vector<common_adapter_lora_info> parse_lora_request(875 const std::vector<common_adapter_lora_info> & lora_base,876 const json & data) {877 std::vector<common_adapter_lora_info> lora(lora_base);878 int max_idx = lora.size();879 880 // clear existing value881 for (auto & entry : lora) {882 entry.scale = 0.0f;883 }884 885 // set value886 for (const auto & entry : data) {887 int id = json_value(entry, "id", -1);888 float scale = json_value(entry, "scale", 0.0f);889 if (0 <= id && id < max_idx) {890 lora[id].scale = scale;891 } else {892 throw std::runtime_error("invalid adapter id");893 }894 }895 896 return lora;897}898