KBaba7/llama.cpp
0
1#include "utils.hpp"2 3#include "arg.h"4#include "common.h"5#include "json-schema-to-grammar.h"6#include "llama.h"7#include "log.h"8#include "sampling.h"9#include "speculative.h"10 11// Change JSON_ASSERT from assert() to GGML_ASSERT:12#define JSON_ASSERT GGML_ASSERT13#include "json.hpp"14// mime type for sending response15#define MIMETYPE_JSON "application/json; charset=utf-8"16 17// auto generated files (see README.md for details)18#include "index.html.gz.hpp"19#include "loading.html.hpp"20 21#include <atomic>22#include <chrono>23#include <condition_variable>24#include <cstddef>25#include <cinttypes>26#include <deque>27#include <memory>28#include <mutex>29#include <signal.h>30#include <thread>31#include <unordered_map>32#include <unordered_set>33 34using json = nlohmann::ordered_json;35 36constexpr int HTTP_POLLING_SECONDS = 1;37 38enum stop_type {39 STOP_TYPE_NONE,40 STOP_TYPE_EOS,41 STOP_TYPE_WORD,42 STOP_TYPE_LIMIT,43};44 45// state diagram: https://github.com/ggerganov/llama.cpp/pull/928346enum slot_state {47 SLOT_STATE_IDLE,48 SLOT_STATE_STARTED, // TODO: this state is only used for setting up the initial prompt processing; maybe merge it with launch_slot_with_task in the future49 SLOT_STATE_PROCESSING_PROMPT,50 SLOT_STATE_DONE_PROMPT,51 SLOT_STATE_GENERATING,52};53 54enum server_state {55 SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet56 SERVER_STATE_READY, // Server is ready and model is loaded57};58 59enum server_task_type {60 SERVER_TASK_TYPE_COMPLETION,61 SERVER_TASK_TYPE_EMBEDDING,62 SERVER_TASK_TYPE_RERANK,63 SERVER_TASK_TYPE_INFILL,64 SERVER_TASK_TYPE_CANCEL,65 SERVER_TASK_TYPE_NEXT_RESPONSE,66 SERVER_TASK_TYPE_METRICS,67 SERVER_TASK_TYPE_SLOT_SAVE,68 SERVER_TASK_TYPE_SLOT_RESTORE,69 SERVER_TASK_TYPE_SLOT_ERASE,70 SERVER_TASK_TYPE_SET_LORA,71};72 73enum oaicompat_type {74 OAICOMPAT_TYPE_NONE,75 OAICOMPAT_TYPE_CHAT,76 OAICOMPAT_TYPE_COMPLETION,77 OAICOMPAT_TYPE_EMBEDDING,78};79 80// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/1181enum error_type {82 ERROR_TYPE_INVALID_REQUEST,83 ERROR_TYPE_AUTHENTICATION,84 ERROR_TYPE_SERVER,85 ERROR_TYPE_NOT_FOUND,86 ERROR_TYPE_PERMISSION,87 ERROR_TYPE_UNAVAILABLE, // custom error88 ERROR_TYPE_NOT_SUPPORTED, // custom error89};90 91struct slot_params {92 bool stream = true;93 bool cache_prompt = true; // remember the prompt to avoid reprocessing all prompt94 bool return_tokens = false;95 96 int32_t n_keep = 0; // number of tokens to keep from initial prompt97 int32_t n_discard = 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half98 int32_t n_predict = -1; // new tokens to predict99 int32_t n_indent = 0; // mininum line indentation for the generated text in number of whitespace characters100 101 int64_t t_max_prompt_ms = -1; // TODO: implement102 int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit103 104 std::vector<common_adapter_lora_info> lora;105 106 std::vector<std::string> antiprompt;107 std::vector<std::string> response_fields;108 bool timings_per_token = false;109 bool post_sampling_probs = false;110 bool ignore_eos = false;111 112 struct common_params_sampling sampling;113 struct common_params_speculative speculative;114 115 // OAI-compat fields116 bool verbose = false;117 oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;118 std::string oaicompat_model;119 std::string oaicompat_cmpl_id;120 common_chat_format oaicompat_chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;121 122 json to_json() const {123 std::vector<std::string> samplers;124 samplers.reserve(sampling.samplers.size());125 for (const auto & sampler : sampling.samplers) {126 samplers.emplace_back(common_sampler_type_to_str(sampler));127 }128 129 json lora = json::array();130 for (size_t i = 0; i < this->lora.size(); ++i) {131 lora.push_back({{"id", i}, {"scale", this->lora[i].scale}});132 }133 134 std::vector<std::string> grammar_trigger_words;135 for (const auto & trigger : sampling.grammar_trigger_words) {136 grammar_trigger_words.push_back(trigger.word);137 }138 139 return json {140 {"n_predict", n_predict}, // Server configured n_predict141 {"seed", sampling.seed},142 {"temperature", sampling.temp},143 {"dynatemp_range", sampling.dynatemp_range},144 {"dynatemp_exponent", sampling.dynatemp_exponent},145 {"top_k", sampling.top_k},146 {"top_p", sampling.top_p},147 {"min_p", sampling.min_p},148 {"xtc_probability", sampling.xtc_probability},149 {"xtc_threshold", sampling.xtc_threshold},150 {"typical_p", sampling.typ_p},151 {"repeat_last_n", sampling.penalty_last_n},152 {"repeat_penalty", sampling.penalty_repeat},153 {"presence_penalty", sampling.penalty_present},154 {"frequency_penalty", sampling.penalty_freq},155 {"dry_multiplier", sampling.dry_multiplier},156 {"dry_base", sampling.dry_base},157 {"dry_allowed_length", sampling.dry_allowed_length},158 {"dry_penalty_last_n", sampling.dry_penalty_last_n},159 {"dry_sequence_breakers", sampling.dry_sequence_breakers},160 {"mirostat", sampling.mirostat},161 {"mirostat_tau", sampling.mirostat_tau},162 {"mirostat_eta", sampling.mirostat_eta},163 {"stop", antiprompt},164 {"max_tokens", n_predict}, // User configured n_predict165 {"n_keep", n_keep},166 {"n_discard", n_discard},167 {"ignore_eos", sampling.ignore_eos},168 {"stream", stream},169 {"logit_bias", format_logit_bias(sampling.logit_bias)},170 {"n_probs", sampling.n_probs},171 {"min_keep", sampling.min_keep},172 {"grammar", sampling.grammar},173 {"grammar_trigger_words", grammar_trigger_words},174 {"grammar_trigger_tokens", sampling.grammar_trigger_tokens},175 {"preserved_tokens", sampling.preserved_tokens},176 {"samplers", samplers},177 {"speculative.n_max", speculative.n_max},178 {"speculative.n_min", speculative.n_min},179 {"speculative.p_min", speculative.p_min},180 {"timings_per_token", timings_per_token},181 {"post_sampling_probs", post_sampling_probs},182 {"lora", lora},183 };184 }185};186 187struct server_task {188 int id = -1; // to be filled by server_queue189 int index = -1; // used when there are multiple prompts (batch request)190 191 server_task_type type;192 193 // used by SERVER_TASK_TYPE_CANCEL194 int id_target = -1;195 196 // used by SERVER_TASK_TYPE_INFERENCE197 slot_params params;198 llama_tokens prompt_tokens;199 int id_selected_slot = -1;200 201 // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE202 struct slot_action {203 int slot_id;204 std::string filename;205 std::string filepath;206 };207 slot_action slot_action;208 209 // used by SERVER_TASK_TYPE_METRICS210 bool metrics_reset_bucket = false;211 212 // used by SERVER_TASK_TYPE_SET_LORA213 std::vector<common_adapter_lora_info> set_lora;214 215 server_task(server_task_type type) : type(type) {}216 217 static slot_params params_from_json_cmpl(218 const llama_context * ctx,219 const common_params & params_base,220 const json & data) {221 const llama_model * model = llama_get_model(ctx);222 const llama_vocab * vocab = llama_model_get_vocab(model);223 224 slot_params params;225 226 // Sampling parameter defaults are loaded from the global server context (but individual requests can still override them)227 slot_params defaults;228 defaults.sampling = params_base.sampling;229 defaults.speculative = params_base.speculative;230 231 // enabling this will output extra debug information in the HTTP responses from the server232 params.verbose = params_base.verbosity > 9;233 params.timings_per_token = json_value(data, "timings_per_token", false);234 235 params.stream = json_value(data, "stream", false);236 params.cache_prompt = json_value(data, "cache_prompt", true);237 params.return_tokens = json_value(data, "return_tokens", false);238 params.n_predict = json_value(data, "n_predict", json_value(data, "max_tokens", defaults.n_predict));239 params.n_indent = json_value(data, "n_indent", defaults.n_indent);240 params.n_keep = json_value(data, "n_keep", defaults.n_keep);241 params.n_discard = json_value(data, "n_discard", defaults.n_discard);242 //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement243 params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms);244 params.response_fields = json_value(data, "response_fields", std::vector<std::string>());245 246 params.sampling.top_k = json_value(data, "top_k", defaults.sampling.top_k);247 params.sampling.top_p = json_value(data, "top_p", defaults.sampling.top_p);248 params.sampling.min_p = json_value(data, "min_p", defaults.sampling.min_p);249 params.sampling.xtc_probability = json_value(data, "xtc_probability", defaults.sampling.xtc_probability);250 params.sampling.xtc_threshold = json_value(data, "xtc_threshold", defaults.sampling.xtc_threshold);251 params.sampling.typ_p = json_value(data, "typical_p", defaults.sampling.typ_p);252 params.sampling.temp = json_value(data, "temperature", defaults.sampling.temp);253 params.sampling.dynatemp_range = json_value(data, "dynatemp_range", defaults.sampling.dynatemp_range);254 params.sampling.dynatemp_exponent = json_value(data, "dynatemp_exponent", defaults.sampling.dynatemp_exponent);255 params.sampling.penalty_last_n = json_value(data, "repeat_last_n", defaults.sampling.penalty_last_n);256 params.sampling.penalty_repeat = json_value(data, "repeat_penalty", defaults.sampling.penalty_repeat);257 params.sampling.penalty_freq = json_value(data, "frequency_penalty", defaults.sampling.penalty_freq);258 params.sampling.penalty_present = json_value(data, "presence_penalty", defaults.sampling.penalty_present);259 params.sampling.dry_multiplier = json_value(data, "dry_multiplier", defaults.sampling.dry_multiplier);260 params.sampling.dry_base = json_value(data, "dry_base", defaults.sampling.dry_base);261 params.sampling.dry_allowed_length = json_value(data, "dry_allowed_length", defaults.sampling.dry_allowed_length);262 params.sampling.dry_penalty_last_n = json_value(data, "dry_penalty_last_n", defaults.sampling.dry_penalty_last_n);263 params.sampling.mirostat = json_value(data, "mirostat", defaults.sampling.mirostat);264 params.sampling.mirostat_tau = json_value(data, "mirostat_tau", defaults.sampling.mirostat_tau);265 params.sampling.mirostat_eta = json_value(data, "mirostat_eta", defaults.sampling.mirostat_eta);266 params.sampling.seed = json_value(data, "seed", defaults.sampling.seed);267 params.sampling.n_probs = json_value(data, "n_probs", defaults.sampling.n_probs);268 params.sampling.min_keep = json_value(data, "min_keep", defaults.sampling.min_keep);269 params.post_sampling_probs = json_value(data, "post_sampling_probs", defaults.post_sampling_probs);270 271 params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min);272 params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max);273 params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min);274 275 params.speculative.n_min = std::min(params.speculative.n_max, params.speculative.n_min);276 params.speculative.n_min = std::max(params.speculative.n_min, 2);277 params.speculative.n_max = std::max(params.speculative.n_max, 0);278 279 // Use OpenAI API logprobs only if n_probs wasn't provided280 if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs){281 params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs);282 }283 284 if (data.contains("lora")) {285 if (data.at("lora").is_array()) {286 params.lora = parse_lora_request(params_base.lora_adapters, data.at("lora"));287 } else {288 throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields");289 }290 } else {291 params.lora = params_base.lora_adapters;292 }293 294 // TODO: add more sanity checks for the input parameters295 296 if (params.sampling.penalty_last_n < -1) {297 throw std::runtime_error("Error: repeat_last_n must be >= -1");298 }299 300 if (params.sampling.dry_penalty_last_n < -1) {301 throw std::runtime_error("Error: dry_penalty_last_n must be >= -1");302 }303 304 if (params.sampling.penalty_last_n == -1) {305 // note: should be the slot's context and not the full context, but it's ok306 params.sampling.penalty_last_n = llama_n_ctx(ctx);307 }308 309 if (params.sampling.dry_penalty_last_n == -1) {310 params.sampling.dry_penalty_last_n = llama_n_ctx(ctx);311 }312 313 if (params.sampling.dry_base < 1.0f) {314 params.sampling.dry_base = defaults.sampling.dry_base;315 }316 317 // sequence breakers for DRY318 {319 // Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format320 // Ref: https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39321 322 if (data.contains("dry_sequence_breakers")) {323 params.sampling.dry_sequence_breakers = json_value(data, "dry_sequence_breakers", std::vector<std::string>());324 if (params.sampling.dry_sequence_breakers.empty()) {325 throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings");326 }327 }328 }329 330 // process "json_schema" and "grammar"331 if (data.contains("json_schema") && !data.at("json_schema").is_null() && data.contains("grammar") && !data.at("grammar").is_null()) {332 throw std::runtime_error("Either \"json_schema\" or \"grammar\" can be specified, but not both");333 }334 if (data.contains("json_schema") && !data.contains("grammar")) {335 try {336 auto schema = json_value(data, "json_schema", json::object());337 LOG_DBG("JSON schema: %s\n", schema.dump(2).c_str());338 params.sampling.grammar = json_schema_to_grammar(schema);339 LOG_DBG("Converted grammar: %s\n", params.sampling.grammar.c_str());340 } catch (const std::exception & e) {341 throw std::runtime_error(std::string("\"json_schema\": ") + e.what());342 }343 } else {344 params.sampling.grammar = json_value(data, "grammar", defaults.sampling.grammar);345 LOG_DBG("Grammar: %s\n", params.sampling.grammar.c_str());346 params.sampling.grammar_lazy = json_value(data, "grammar_lazy", defaults.sampling.grammar_lazy);347 LOG_DBG("Grammar lazy: %s\n", params.sampling.grammar_lazy ? "true" : "false");348 }349 350 {351 auto it = data.find("chat_format");352 if (it != data.end()) {353 params.oaicompat_chat_format = static_cast<common_chat_format>(it->get<int>());354 LOG_INF("Chat format: %s\n", common_chat_format_name(params.oaicompat_chat_format).c_str());355 } else {356 params.oaicompat_chat_format = defaults.oaicompat_chat_format;357 }358 }359 360 {361 const auto grammar_triggers = data.find("grammar_triggers");362 if (grammar_triggers != data.end()) {363 for (const auto & t : *grammar_triggers) {364 common_grammar_trigger trigger;365 trigger.word = t.at("word");366 trigger.at_start = t.at("at_start");367 368 auto ids = common_tokenize(vocab, trigger.word, /* add_special= */ false, /* parse_special= */ true);369 if (ids.size() == 1) {370 LOG_DBG("Grammar trigger token: %d (`%s`)\n", ids[0], trigger.word.c_str());371 params.sampling.grammar_trigger_tokens.push_back(ids[0]);372 params.sampling.preserved_tokens.insert(ids[0]);373 continue;374 }375 LOG_DBG("Grammar trigger word: `%s`\n", trigger.word.c_str());376 params.sampling.grammar_trigger_words.push_back(trigger);377 }378 }379 const auto preserved_tokens = data.find("preserved_tokens");380 if (preserved_tokens != data.end()) {381 for (const auto & t : *preserved_tokens) {382 auto ids = common_tokenize(vocab, t.get<std::string>(), /* add_special= */ false, /* parse_special= */ true);383 if (ids.size() == 1) {384 LOG_DBG("Preserved token: %d\n", ids[0]);385 params.sampling.preserved_tokens.insert(ids[0]);386 } else {387 // This may happen when using a tool call style meant for a model with special tokens to preserve on a model without said tokens.388 LOG_WRN("Not preserved because more than 1 token (wrong chat template override?): %s\n", t.get<std::string>().c_str());389 }390 }391 }392 if (params.sampling.grammar_lazy) {393 GGML_ASSERT(params.sampling.grammar_trigger_tokens.size() > 0 || params.sampling.grammar_trigger_words.size() > 0);394 }395 }396 397 {398 params.sampling.logit_bias.clear();399 params.ignore_eos = json_value(data, "ignore_eos", false);400 401 const auto & logit_bias = data.find("logit_bias");402 if (logit_bias != data.end() && logit_bias->is_array()) {403 const int n_vocab = llama_vocab_n_tokens(vocab);404 for (const auto & el : *logit_bias) {405 // TODO: we may want to throw errors here, in case "el" is incorrect406 if (el.is_array() && el.size() == 2) {407 float bias;408 if (el[1].is_number()) {409 bias = el[1].get<float>();410 } else if (el[1].is_boolean() && !el[1].get<bool>()) {411 bias = -INFINITY;412 } else {413 continue;414 }415 416 if (el[0].is_number_integer()) {417 llama_token tok = el[0].get<llama_token>();418 if (tok >= 0 && tok < n_vocab) {419 params.sampling.logit_bias.push_back({tok, bias});420 }421 } else if (el[0].is_string()) {422 auto toks = common_tokenize(vocab, el[0].get<std::string>(), false);423 for (auto tok : toks) {424 params.sampling.logit_bias.push_back({tok, bias});425 }426 }427 }428 }429 }430 }431 432 {433 params.antiprompt.clear();434 435 const auto & stop = data.find("stop");436 if (stop != data.end() && stop->is_array()) {437 for (const auto & word : *stop) {438 if (!word.empty()) {439 params.antiprompt.push_back(word);440 }441 }442 }443 }444 445 {446 const auto samplers = data.find("samplers");447 if (samplers != data.end()) {448 if (samplers->is_array()) {449 params.sampling.samplers = common_sampler_types_from_names(*samplers, false);450 } else if (samplers->is_string()){451 params.sampling.samplers = common_sampler_types_from_chars(samplers->get<std::string>());452 }453 } else {454 params.sampling.samplers = defaults.sampling.samplers;455 }456 }457 458 std::string model_name = params_base.model_alias.empty() ? DEFAULT_OAICOMPAT_MODEL : params_base.model_alias;459 params.oaicompat_model = json_value(data, "model", model_name);460 461 return params;462 }463 464 // utility function465 static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {466 std::unordered_set<int> ids(tasks.size());467 for (size_t i = 0; i < tasks.size(); i++) {468 ids.insert(tasks[i].id);469 }470 return ids;471 }472};473 474struct result_timings {475 int32_t prompt_n = -1;476 double prompt_ms;477 double prompt_per_token_ms;478 double prompt_per_second;479 480 int32_t predicted_n = -1;481 double predicted_ms;482 double predicted_per_token_ms;483 double predicted_per_second;484 485 json to_json() const {486 return {487 {"prompt_n", prompt_n},488 {"prompt_ms", prompt_ms},489 {"prompt_per_token_ms", prompt_per_token_ms},490 {"prompt_per_second", prompt_per_second},491 492 {"predicted_n", predicted_n},493 {"predicted_ms", predicted_ms},494 {"predicted_per_token_ms", predicted_per_token_ms},495 {"predicted_per_second", predicted_per_second},496 };497 }498};499 500struct server_task_result {501 int id = -1;502 int id_slot = -1;503 virtual bool is_error() {504 // only used by server_task_result_error505 return false;506 }507 virtual bool is_stop() {508 // only used by server_task_result_cmpl_*509 return false;510 }511 virtual int get_index() {512 return -1;513 }514 virtual json to_json() = 0;515 virtual ~server_task_result() = default;516};517 518// using shared_ptr for polymorphism of server_task_result519using server_task_result_ptr = std::unique_ptr<server_task_result>;520 521inline std::string stop_type_to_str(stop_type type) {522 switch (type) {523 case STOP_TYPE_EOS: return "eos";524 case STOP_TYPE_WORD: return "word";525 case STOP_TYPE_LIMIT: return "limit";526 default: return "none";527 }528}529 530struct completion_token_output {531 llama_token tok;532 float prob;533 std::string text_to_send;534 struct prob_info {535 llama_token tok;536 std::string txt;537 float prob;538 };539 std::vector<prob_info> probs;540 541 json to_json(bool post_sampling_probs) const {542 json probs_for_token = json::array();543 for (const auto & p : probs) {544 std::string txt(p.txt);545 txt.resize(validate_utf8(txt));546 probs_for_token.push_back(json {547 {"id", p.tok},548 {"token", txt},549 {"bytes", str_to_bytes(p.txt)},550 {551 post_sampling_probs ? "prob" : "logprob",552 post_sampling_probs ? p.prob : logarithm(p.prob)553 },554 });555 }556 return probs_for_token;557 }558 559 static json probs_vector_to_json(const std::vector<completion_token_output> & probs, bool post_sampling_probs) {560 json out = json::array();561 for (const auto & p : probs) {562 std::string txt(p.text_to_send);563 txt.resize(validate_utf8(txt));564 out.push_back(json {565 {"id", p.tok},566 {"token", txt},567 {"bytes", str_to_bytes(p.text_to_send)},568 {569 post_sampling_probs ? "prob" : "logprob",570 post_sampling_probs ? p.prob : logarithm(p.prob)571 },572 {573 post_sampling_probs ? "top_probs" : "top_logprobs",574 p.to_json(post_sampling_probs)575 },576 });577 }578 return out;579 }580 581 static float logarithm(float x) {582 // nlohmann::json converts -inf to null, so we need to prevent that583 return x == 0.0f ? std::numeric_limits<float>::lowest() : std::log(x);584 }585 586 static std::vector<unsigned char> str_to_bytes(const std::string & str) {587 std::vector<unsigned char> bytes;588 for (unsigned char c : str) {589 bytes.push_back(c);590 }591 return bytes;592 }593};594 595struct server_task_result_cmpl_final : server_task_result {596 int index = 0;597 598 std::string content;599 llama_tokens tokens;600 601 bool stream;602 result_timings timings;603 std::string prompt;604 605 bool truncated;606 int32_t n_decoded;607 int32_t n_prompt_tokens;608 int32_t n_tokens_cached;609 bool has_new_line;610 std::string stopping_word;611 stop_type stop = STOP_TYPE_NONE;612 613 bool post_sampling_probs;614 std::vector<completion_token_output> probs_output;615 std::vector<std::string> response_fields;616 617 slot_params generation_params;618 619 // OAI-compat fields620 bool verbose = false;621 oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;622 std::string oaicompat_model;623 std::string oaicompat_cmpl_id;624 common_chat_format oaicompat_chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;625 626 virtual int get_index() override {627 return index;628 }629 630 virtual bool is_stop() override {631 return true; // in stream mode, final responses are considered stop632 }633 634 virtual json to_json() override {635 switch (oaicompat) {636 case OAICOMPAT_TYPE_NONE:637 return to_json_non_oaicompat();638 case OAICOMPAT_TYPE_COMPLETION:639 return to_json_oaicompat();640 case OAICOMPAT_TYPE_CHAT:641 return stream ? to_json_oaicompat_chat_stream() : to_json_oaicompat_chat();642 default:643 GGML_ASSERT(false && "Invalid oaicompat_type");644 }645 }646 647 json to_json_non_oaicompat() {648 json res = json {649 {"index", index},650 {"content", stream ? "" : content}, // in stream mode, content is already in last partial chunk651 {"tokens", stream ? llama_tokens {} : tokens},652 {"id_slot", id_slot},653 {"stop", true},654 {"model", oaicompat_model},655 {"tokens_predicted", n_decoded},656 {"tokens_evaluated", n_prompt_tokens},657 {"generation_settings", generation_params.to_json()},658 {"prompt", prompt},659 {"has_new_line", has_new_line},660 {"truncated", truncated},661 {"stop_type", stop_type_to_str(stop)},662 {"stopping_word", stopping_word},663 {"tokens_cached", n_tokens_cached},664 {"timings", timings.to_json()},665 };666 if (!stream && !probs_output.empty()) {667 res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);668 }669 return response_fields.empty() ? res : json_get_nested_values(response_fields, res);670 }671 672 json to_json_oaicompat() {673 std::time_t t = std::time(0);674 json logprobs = json(nullptr); // OAI default to null675 if (!stream && probs_output.size() > 0) {676 logprobs = json{677 {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},678 };679 }680 json finish_reason = "length";681 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {682 finish_reason = "stop";683 }684 json res = json {685 {"choices", json::array({686 json{687 {"text", stream ? "" : content}, // in stream mode, content is already in last partial chunk688 {"index", index},689 {"logprobs", logprobs},690 {"finish_reason", finish_reason},691 }692 })},693 {"created", t},694 {"model", oaicompat_model},695 {"system_fingerprint", build_info},696 {"object", "text_completion"},697 {"usage", json {698 {"completion_tokens", n_decoded},699 {"prompt_tokens", n_prompt_tokens},700 {"total_tokens", n_decoded + n_prompt_tokens}701 }},702 {"id", oaicompat_cmpl_id}703 };704 705 // extra fields for debugging purposes706 if (verbose) {707 res["__verbose"] = to_json_non_oaicompat();708 }709 if (timings.prompt_n >= 0) {710 res.push_back({"timings", timings.to_json()});711 }712 713 return res;714 }715 716 json to_json_oaicompat_chat() {717 std::string finish_reason = "length";718 common_chat_msg msg;719 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {720 LOG_DBG("Parsing chat message: %s\n", content.c_str());721 msg = common_chat_parse(content, oaicompat_chat_format);722 finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";723 } else {724 msg.content = content;725 }726 727 json tool_calls;728 if (!msg.tool_calls.empty()) {729 tool_calls = json::array();730 for (const auto & tc : msg.tool_calls) {731 tool_calls.push_back({732 {"type", "function"},733 {"function", {734 {"name", tc.name},735 {"arguments", tc.arguments},736 }},737 {"id", tc.id},738 });739 }740 }741 742 json message {743 {"content", msg.content},744 {"tool_calls", tool_calls},745 {"role", "assistant"},746 };747 if (!msg.tool_plan.empty()) {748 message["tool_plan"] = msg.tool_plan;749 }750 751 json choice {752 {"finish_reason", finish_reason},753 {"index", 0},754 {"message", message},755 };756 757 if (!stream && probs_output.size() > 0) {758 choice["logprobs"] = json{759 {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},760 };761 }762 763 std::time_t t = std::time(0);764 765 json res = json {766 {"choices", json::array({choice})},767 {"created", t},768 {"model", oaicompat_model},769 {"system_fingerprint", build_info},770 {"object", "chat.completion"},771 {"usage", json {772 {"completion_tokens", n_decoded},773 {"prompt_tokens", n_prompt_tokens},774 {"total_tokens", n_decoded + n_prompt_tokens}775 }},776 {"id", oaicompat_cmpl_id}777 };778 779 // extra fields for debugging purposes780 if (verbose) {781 res["__verbose"] = to_json_non_oaicompat();782 }783 if (timings.prompt_n >= 0) {784 res.push_back({"timings", timings.to_json()});785 }786 787 return res;788 }789 790 json to_json_oaicompat_chat_stream() {791 std::time_t t = std::time(0);792 std::string finish_reason = "length";793 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {794 finish_reason = "stop";795 }796 797 json choice = json {798 {"finish_reason", finish_reason},799 {"index", 0},800 {"delta", json::object()}801 };802 803 json ret = json {804 {"choices", json::array({choice})},805 {"created", t},806 {"id", oaicompat_cmpl_id},807 {"model", oaicompat_model},808 {"system_fingerprint", build_info},809 {"object", "chat.completion.chunk"},810 {"usage", json {811 {"completion_tokens", n_decoded},812 {"prompt_tokens", n_prompt_tokens},813 {"total_tokens", n_decoded + n_prompt_tokens},814 }},815 };816 817 if (timings.prompt_n >= 0) {818 ret.push_back({"timings", timings.to_json()});819 }820 821 return ret;822 }823};824 825struct server_task_result_cmpl_partial : server_task_result {826 int index = 0;827 828 std::string content;829 llama_tokens tokens;830 831 int32_t n_decoded;832 int32_t n_prompt_tokens;833 834 bool post_sampling_probs;835 completion_token_output prob_output;836 result_timings timings;837 838 // OAI-compat fields839 bool verbose = false;840 oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;841 std::string oaicompat_model;842 std::string oaicompat_cmpl_id;843 844 virtual int get_index() override {845 return index;846 }847 848 virtual bool is_stop() override {849 return false; // in stream mode, partial responses are not considered stop850 }851 852 virtual json to_json() override {853 switch (oaicompat) {854 case OAICOMPAT_TYPE_NONE:855 return to_json_non_oaicompat();856 case OAICOMPAT_TYPE_COMPLETION:857 return to_json_oaicompat();858 case OAICOMPAT_TYPE_CHAT:859 return to_json_oaicompat_chat();860 default:861 GGML_ASSERT(false && "Invalid oaicompat_type");862 }863 }864 865 json to_json_non_oaicompat() {866 // non-OAI-compat JSON867 json res = json {868 {"index", index},869 {"content", content},870 {"tokens", tokens},871 {"stop", false},872 {"id_slot", id_slot},873 {"tokens_predicted", n_decoded},874 {"tokens_evaluated", n_prompt_tokens},875 };876 // populate the timings object when needed (usually for the last response or with timings_per_token enabled)877 if (timings.prompt_n > 0) {878 res.push_back({"timings", timings.to_json()});879 }880 if (!prob_output.probs.empty()) {881 res["completion_probabilities"] = completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs);882 }883 return res;884 }885 886 json to_json_oaicompat() {887 std::time_t t = std::time(0);888 json logprobs = json(nullptr); // OAI default to null889 if (prob_output.probs.size() > 0) {890 logprobs = json{891 {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},892 };893 }894 json res = json {895 {"choices", json::array({896 json{897 {"text", content},898 {"index", index},899 {"logprobs", logprobs},900 {"finish_reason", nullptr},901 }902 })},903 {"created", t},904 {"model", oaicompat_model},905 {"system_fingerprint", build_info},906 {"object", "text_completion"},907 {"id", oaicompat_cmpl_id}908 };909 910 // extra fields for debugging purposes911 if (verbose) {912 res["__verbose"] = to_json_non_oaicompat();913 }914 if (timings.prompt_n >= 0) {915 res.push_back({"timings", timings.to_json()});916 }917 918 return res;919 }920 921 json to_json_oaicompat_chat() {922 bool first = n_decoded == 0;923 std::time_t t = std::time(0);924 json choices;925 926 if (first) {927 if (content.empty()) {928 choices = json::array({json{{"finish_reason", nullptr},929 {"index", 0},930 {"delta", json{{"role", "assistant"}}}}});931 } else {932 // We have to send this as two updates to conform to openai behavior933 json initial_ret = json{{"choices", json::array({json{934 {"finish_reason", nullptr},935 {"index", 0},936 {"delta", json{937 {"role", "assistant"}938 }}}})},939 {"created", t},940 {"id", oaicompat_cmpl_id},941 {"model", oaicompat_model},942 {"object", "chat.completion.chunk"}};943 944 json second_ret = json{945 {"choices", json::array({json{{"finish_reason", nullptr},946 {"index", 0},947 {"delta", json {948 {"content", content}}}949 }})},950 {"created", t},951 {"id", oaicompat_cmpl_id},952 {"model", oaicompat_model},953 {"object", "chat.completion.chunk"}};954 955 return std::vector<json>({initial_ret, second_ret});956 }957 } else {958 choices = json::array({json{959 {"finish_reason", nullptr},960 {"index", 0},961 {"delta",962 json {963 {"content", content},964 }},965 }});966 }967 968 GGML_ASSERT(choices.size() >= 1);969 970 if (prob_output.probs.size() > 0) {971 choices[0]["logprobs"] = json{972 {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},973 };974 }975 976 json ret = json {977 {"choices", choices},978 {"created", t},979 {"id", oaicompat_cmpl_id},980 {"model", oaicompat_model},981 {"system_fingerprint", build_info},982 {"object", "chat.completion.chunk"}983 };984 985 if (timings.prompt_n >= 0) {986 ret.push_back({"timings", timings.to_json()});987 }988 989 return std::vector<json>({ret});990 }991};992 993struct server_task_result_embd : server_task_result {994 int index = 0;995 std::vector<std::vector<float>> embedding;996 997 int32_t n_tokens;998 999 // OAI-compat fields1000 oaicompat_type oaicompat = OAICOMPAT_TYPE_NONE;1001 1002 virtual int get_index() override {1003 return index;1004 }1005 1006 virtual json to_json() override {1007 return oaicompat == OAICOMPAT_TYPE_EMBEDDING1008 ? to_json_oaicompat()1009 : to_json_non_oaicompat();1010 }1011 1012 json to_json_non_oaicompat() {1013 return json {1014 {"index", index},1015 {"embedding", embedding},1016 };1017 }1018 1019 json to_json_oaicompat() {1020 return json {1021 {"index", index},1022 {"embedding", embedding[0]},1023 {"tokens_evaluated", n_tokens},1024 };1025 }1026};1027 1028struct server_task_result_rerank : server_task_result {1029 int index = 0;1030 float score = -1e6;1031 1032 int32_t n_tokens;1033 1034 virtual int get_index() override {1035 return index;1036 }1037 1038 virtual json to_json() override {1039 return json {1040 {"index", index},1041 {"score", score},1042 {"tokens_evaluated", n_tokens},1043 };1044 }1045};1046 1047// this function maybe used outside of server_task_result_error1048static json format_error_response(const std::string & message, const enum error_type type) {1049 std::string type_str;1050 int code = 500;1051 switch (type) {1052 case ERROR_TYPE_INVALID_REQUEST:1053 type_str = "invalid_request_error";1054 code = 400;1055 break;1056 case ERROR_TYPE_AUTHENTICATION:1057 type_str = "authentication_error";1058 code = 401;1059 break;1060 case ERROR_TYPE_NOT_FOUND:1061 type_str = "not_found_error";1062 code = 404;1063 break;1064 case ERROR_TYPE_SERVER:1065 type_str = "server_error";1066 code = 500;1067 break;1068 case ERROR_TYPE_PERMISSION:1069 type_str = "permission_error";1070 code = 403;1071 break;1072 case ERROR_TYPE_NOT_SUPPORTED:1073 type_str = "not_supported_error";1074 code = 501;1075 break;1076 case ERROR_TYPE_UNAVAILABLE:1077 type_str = "unavailable_error";1078 code = 503;1079 break;1080 }1081 return json {1082 {"code", code},1083 {"message", message},1084 {"type", type_str},1085 };1086}1087 1088struct server_task_result_error : server_task_result {1089 int index = 0;1090 error_type err_type = ERROR_TYPE_SERVER;1091 std::string err_msg;1092 1093 virtual bool is_error() override {1094 return true;1095 }1096 1097 virtual json to_json() override {1098 return format_error_response(err_msg, err_type);1099 }1100};1101 1102struct server_task_result_metrics : server_task_result {1103 int n_idle_slots;1104 int n_processing_slots;1105 int n_tasks_deferred;1106 int64_t t_start;1107 1108 int32_t kv_cache_tokens_count;1109 int32_t kv_cache_used_cells;1110 1111 // TODO: somehow reuse server_metrics in the future, instead of duplicating the fields1112 uint64_t n_prompt_tokens_processed_total = 0;1113 uint64_t t_prompt_processing_total = 0;1114 uint64_t n_tokens_predicted_total = 0;1115 uint64_t t_tokens_generation_total = 0;1116 1117 uint64_t n_prompt_tokens_processed = 0;1118 uint64_t t_prompt_processing = 0;1119 1120 uint64_t n_tokens_predicted = 0;1121 uint64_t t_tokens_generation = 0;1122 1123 uint64_t n_decode_total = 0;1124 uint64_t n_busy_slots_total = 0;1125 1126 // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy1127 // therefore, we use json to temporarily store the slot.to_json() result1128 json slots_data = json::array();1129 1130 virtual json to_json() override {1131 return json {1132 { "idle", n_idle_slots },1133 { "processing", n_processing_slots },1134 { "deferred", n_tasks_deferred },1135 { "t_start", t_start },1136 1137 { "n_prompt_tokens_processed_total", n_prompt_tokens_processed_total },1138 { "t_tokens_generation_total", t_tokens_generation_total },1139 { "n_tokens_predicted_total", n_tokens_predicted_total },1140 { "t_prompt_processing_total", t_prompt_processing_total },1141 1142 { "n_prompt_tokens_processed", n_prompt_tokens_processed },1143 { "t_prompt_processing", t_prompt_processing },1144 { "n_tokens_predicted", n_tokens_predicted },1145 { "t_tokens_generation", t_tokens_generation },1146 1147 { "n_decode_total", n_decode_total },1148 { "n_busy_slots_total", n_busy_slots_total },1149 1150 { "kv_cache_tokens_count", kv_cache_tokens_count },1151 { "kv_cache_used_cells", kv_cache_used_cells },1152 1153 { "slots", slots_data },1154 };1155 }1156};1157 1158struct server_task_result_slot_save_load : server_task_result {1159 std::string filename;1160 bool is_save; // true = save, false = load1161 1162 size_t n_tokens;1163 size_t n_bytes;1164 double t_ms;1165 1166 virtual json to_json() override {1167 if (is_save) {1168 return json {1169 { "id_slot", id_slot },1170 { "filename", filename },1171 { "n_saved", n_tokens },1172 { "n_written", n_bytes },1173 { "timings", {1174 { "save_ms", t_ms }1175 }},1176 };1177 } else {1178 return json {1179 { "id_slot", id_slot },1180 { "filename", filename },1181 { "n_restored", n_tokens },1182 { "n_read", n_bytes },1183 { "timings", {1184 { "restore_ms", t_ms }1185 }},1186 };1187 }1188 }1189};1190 1191struct server_task_result_slot_erase : server_task_result {1192 size_t n_erased;1193 1194 virtual json to_json() override {1195 return json {1196 { "id_slot", id_slot },1197 { "n_erased", n_erased },1198 };1199 }1200};