Brunobkr/llama.cpp_AlgMor24_github

1#include "server-schema.h"2 3#include "json-schema-to-grammar.h"4 5namespace server_schema {6 7//8// llama.cpp-specific completion schema9//10 11std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params & params_base, task_params & params) {12    std::vector<std::unique_ptr<field>> fields;13    auto add = [&](field * f) {14        fields.emplace_back(f);15    };16 17    add((new field_bool("verbose", params.verbose))18        ->set_desc("Include __verbose field in the response with additional debug information"));19 20    add((new field_bool("timings_per_token", params.timings_per_token))21        ->set_desc("Include prompt processing and text generation speed information in each response"));22 23    add((new field_bool("stream", params.stream))24        ->set_desc("Allows receiving each predicted token in real-time instead of waiting for the completion to finish"));25 26    add((new field_nested("stream_options"))27        ->add_subfield((new field_bool("include_usage", params.include_usage))28            ->set_desc("Whether to include usage information in the stream"))29        ->set_desc("Additional options for streaming responses"));30 31    add((new field_bool("cache_prompt", params.cache_prompt))32        ->set_desc("Re-use KV cache from a previous request if possible. This way the common prefix does not have to be re-processed, only the suffix that differs between the requests"));33 34    add((new field_bool("return_tokens", params.return_tokens))35        ->set_desc("Return the raw generated token ids in the `tokens` field"));36 37    add((new field_bool("return_progress", params.return_progress))38        ->set_desc("Include prompt processing progress events in stream mode"));39 40    add((new field_num("sse_ping_interval", params.sse_ping_interval))41        ->set_hard_limits(-1, INT32_MAX)42        ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings"));43 44    add((new field_num("n_predict", params.n_predict))45        ->set_hard_limits(-1, INT32_MAX)46        ->add_alias("max_completion_tokens")47        ->add_alias("max_tokens")48        ->set_desc("Set the maximum number of tokens to predict. When 0, no tokens will be generated but the prompt is evaluated into the cache"));49 50    add((new field_num("n_indent", params.n_indent))51        ->set_hard_limits(0, INT32_MAX)52        ->set_desc("Specify the minimum line indentation for the generated text in number of whitespace characters. Useful for code completion tasks"));53 54    add((new field_num("n_keep", params.n_keep))55        ->set_hard_limits(-1, INT32_MAX)56        ->set_desc("Specify the number of tokens from the initial prompt to retain when context size is exceeded. Use -1 to retain all tokens from the prompt"));57 58    add((new field_num("n_discard", params.n_discard))59        ->set_hard_limits(0, INT32_MAX)60        ->set_desc("Number of tokens after n_keep that may be discarded when shifting context (0 = half context)"));61 62    add((new field_num("n_cmpl", params.n_cmpl))63        ->set_hard_limits(1, params_base.n_parallel)64        ->add_alias("n") // alias "n" as fallback (OpenAI completions API)65        ->set_desc("Number of completions to generate. If the input has multiple prompts, total outputs will be N prompts times n_cmpl"));66 67    add((new field_num("n_cache_reuse", params.n_cache_reuse))68        ->set_hard_limits(0, INT32_MAX)69        ->set_desc("Min chunk size to attempt reusing from the cache via KV shifting. See --cache-reuse arg"));70 71    // TODO: implement t_max_prompt_ms72    // add((new field_num("t_max_prompt_ms", params.t_max_prompt_ms))73 74    add((new field_num("t_max_predict_ms", params.t_max_predict_ms))75        ->set_hard_limits(-1, std::numeric_limits<int64_t>::max())76        ->set_desc("Set a time limit in milliseconds for the prediction phase. The timeout triggers if generation exceeds this time (measured since the first token) and a newline has been generated. Useful for FIM applications"));77 78    add((new field_json("response_fields"))79        ->set_desc("A list of response fields to return. Missing fields are omitted without error. Fields with a slash are unnested (e.g. generation_settings/n_predict moves n_predict to the root)")80        ->set_handler([&](field_eval_context & ctx, const json & data) {81            ctx.params.response_fields = json_value(data, "response_fields", std::vector<std::string>());82        }));83 84 85    //86    // Sampling params87    //88 89    add((new field_num("top_k", params.sampling.top_k))90        ->set_limits(0, INT32_MAX)91        ->set_desc("Limit the next token selection to the K most probable tokens (0 = disabled)"));92 93    add((new field_num("top_p", params.sampling.top_p))94        ->set_limits(0.0f, 1.0f)95        ->set_desc("Limit the next token selection to a subset of tokens with cumulative probability above threshold P (1.0 = disabled)"));96 97    add((new field_num("min_p", params.sampling.min_p))98        ->set_limits(0.0f, 1.0f)99        ->set_desc("The minimum probability for a token to be considered, relative to the probability of the most likely token (0 = disabled)"));100 101    add((new field_num("top_n_sigma", params.sampling.top_n_sigma))102        ->set_desc("Keep tokens within n standard deviations of the top token logit (< 0 = disabled)"));103 104    add((new field_num("xtc_probability", params.sampling.xtc_probability))105        ->set_limits(0.0f, 1.0f)106        ->set_desc("Set the chance for token removal via XTC sampler (0 = disabled)"));107 108    add((new field_num("xtc_threshold", params.sampling.xtc_threshold))109        ->set_limits(0.0f, 1.0f)110        ->set_desc("Set a minimum probability threshold for tokens to be removed via XTC sampler (> 0.5 disables XTC)"));111 112    add((new field_num("typical_p", params.sampling.typ_p))113        // ->set_limits(0.0f, 1.0f) // what's the valid range?114        ->set_desc("Enable locally typical sampling with parameter p (1.0 = disabled)"));115 116    add((new field_num("temperature", params.sampling.temp))117        ->set_limits(0.0f, std::numeric_limits<float>::infinity())118        ->set_desc("Adjust the randomness of the generated text (0 = greedy)"));119 120    add((new field_num("dynatemp_range", params.sampling.dynatemp_range))121        ->set_desc("Dynamic temperature range. The final temperature will be in [temperature - range, temperature + range] (0 = disabled)"));122 123    add((new field_num("dynatemp_exponent", params.sampling.dynatemp_exponent))124        ->set_desc("Dynamic temperature exponent, controls how entropy maps to temperature"));125 126    add((new field_num("repeat_last_n", params.sampling.penalty_last_n))127        ->set_hard_limits(0, INT32_MAX)128        ->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled)"));129 130    add((new field_num("repeat_penalty", params.sampling.penalty_repeat))131        ->set_desc("Control the repetition of token sequences in the generated text (1.0 = disabled)"));132 133    add((new field_num("frequency_penalty", params.sampling.penalty_freq))134        ->set_desc("Repeat alpha frequency penalty (0 = disabled)"));135 136    add((new field_num("presence_penalty", params.sampling.penalty_present))137        ->set_desc("Repeat alpha presence penalty (0 = disabled)"));138 139    add((new field_num("dry_multiplier", params.sampling.dry_multiplier))140        ->set_desc("Set the DRY (Don't Repeat Yourself) repetition penalty multiplier (0 = disabled)"));141 142    add((new field_num("dry_base", params.sampling.dry_base))143        ->set_desc("Set the DRY repetition penalty base value (must be >= 1.0, any values < 1.0 will be replaced with the default value)")144        ->set_handler([&](field_eval_context & ctx, const json & data) {145            float v = data.at("dry_base").get<float>();146            ctx.params.sampling.dry_base = (v < 1.0f) ? params_base.sampling.dry_base : v;147        }));148 149    add((new field_num("dry_allowed_length", params.sampling.dry_allowed_length))150        ->set_hard_limits(0, INT32_MAX)151        ->set_desc("Tokens that extend repetition beyond this length receive exponentially increasing penalty: multiplier * base ^ (sequence_length - allowed_length)"));152 153    add((new field_num("dry_penalty_last_n", params.sampling.dry_penalty_last_n))154        ->set_hard_limits(0, INT32_MAX)155        ->set_desc("How many tokens to scan for repetitions (0 = disabled)"));156 157    add((new field_num("mirostat", params.sampling.mirostat))158        ->set_limits(0, 2)159        ->set_desc("Enable Mirostat sampling, controlling perplexity during text generation (0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)"));160 161    add((new field_num("mirostat_tau", params.sampling.mirostat_tau))162        ->set_desc("Set the Mirostat target entropy, parameter tau"));163 164    add((new field_num("mirostat_eta", params.sampling.mirostat_eta))165        ->set_desc("Set the Mirostat learning rate, parameter eta"));166 167    add((new field_num("adaptive_target", params.sampling.adaptive_target))168        ->set_limits(-std::numeric_limits<float>::max(), 1.0f)169        ->set_desc("Adaptive sampling target entropy (valid range 0.0 to 1.0; negative = disabled)"));170 171    add((new field_num("adaptive_decay", params.sampling.adaptive_decay))172        ->set_hard_limits(0.0f, 0.99f)173        ->set_desc("EMA decay for adaptive sampling; history approximates 1/(1-decay) tokens"));174 175    // seed is uint32_t; field_num uses int32_t so use a handler176    add((new field_num("seed", params.sampling.seed))177        ->set_desc("Set the random number generator (RNG) seed (-1 = random)"));178 179    add((new field_num("n_probs", params.sampling.n_probs))180        ->add_alias("logprobs") // use "logprobs" if "n_probs" wasn't provided181        ->set_desc("If greater than 0, output the probabilities of top N tokens for each generated token"));182 183    add((new field_num("min_keep", params.sampling.min_keep))184        ->set_hard_limits(0, INT32_MAX)185        ->set_desc("If greater than 0, force samplers to return at least N possible tokens"));186 187    add((new field_bool("backend_sampling", params.sampling.backend_sampling))188        ->set_desc("Use backend sampling instead of llama.cpp sampling"));189 190    add((new field_bool("post_sampling_probs", params.post_sampling_probs))191        ->set_desc("Return probabilities of top n_probs tokens after applying the sampling chain"));192 193    //194    // Speculative decoding params195    //196 197    // TODO: to keep things simple, we disable speculative parameter adjustments for now198#if 0199    // TODO: for now, be able to adjust only the draft-model based speculative parameters200    add((new field_num("speculative.n_max", params.speculative.draft.n_max))201        ->set_hard_limits(0, INT32_MAX)202        ->set_desc("Maximum number of tokens to draft during speculative decoding"));203 204    add((new field_num("speculative.n_min", params.speculative.draft.n_min))205        ->set_hard_limits(0, INT32_MAX)206        ->set_desc("Minimum number of draft tokens to use for speculative decoding");207 208    add((new field_num("speculative.p_min", params.speculative.draft.p_min))209        ->set_hard_limits(0.0f, 1.0f)210        ->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));211 212 213    add((new field_str("speculative.type"))214        ->set_desc("Speculative decoding method (for debugging and research purposes)")215        ->set_handler([&](field_eval_context & ctx, const json & data) {216            ctx.params.speculative.types = { common_speculative_type_from_name(data.at("speculative.type").get<std::string>()) };217        }));218 219    add((new field_num("speculative.ngram_size_n", params.speculative.ngram_simple.size_n))220        ->set_desc("Ngram size for lookup in ngram-based speculative decoding"));221 222    add((new field_num("speculative.ngram_size_m", params.speculative.ngram_simple.size_m))223        ->set_desc("Mgram size for speculative tokens in ngram-based speculative decoding"));224 225    add((new field_num("speculative.ngram_min_hits", params.speculative.ngram_simple.min_hits))226        ->set_desc("Minimum hits at ngram lookup for mgram to be proposed"));227#endif228 229    add((new field_json("lora"))230        ->set_desc("A list of LoRA adapters to apply to this request. Each entry must have `id` and `scale` fields. Adapters not listed default to scale 0.0")231        ->set_handler([&](field_eval_context & ctx, const json & data) {232            const auto & lora = data.at("lora");233            if (!lora.is_array()) {234                throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields");235            }236            ctx.params.lora = parse_lora_request(lora);237        }));238 239    // sequence breakers for DRY240    // Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format241    // Ref: https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39242    add((new field_json("dry_sequence_breakers"))243        ->set_desc("Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted")244        ->set_handler([&](field_eval_context & ctx, const json & data) {245            ctx.params.sampling.dry_sequence_breakers = json_value(data, "dry_sequence_breakers", std::vector<std::string>());246            if (ctx.params.sampling.dry_sequence_breakers.empty()) {247                throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings");248            }249        }));250 251    // handle both "json_schema" and "grammar"252    add((new field_json("json_schema"))253        ->add_alias("grammar")254        ->set_desc("Set a JSON schema (json_schema) or GBNF grammar string (grammar) for constrained generation. json_schema takes precedence if both are provided")255        ->set_handler([&](field_eval_context & ctx, const json & data) {256            auto & params = ctx.params;257            if (data.contains("json_schema") && !data.contains("grammar")) {258                try {259                    auto schema                  = json_value(data, "json_schema", json::object());260                    SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str());261                    std::string grammar_str      = json_schema_to_grammar(schema);262                    SRV_DBG("Converted grammar: %s\n", grammar_str.c_str());263                    params.sampling.grammar      = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, std::move(grammar_str)};264                } catch (const std::exception & e) {265                    throw std::runtime_error(std::string("\"json_schema\": ") + e.what());266                }267            } else {268                std::string grammar_str = json_value(data, "grammar", std::string());269                if (!grammar_str.empty()) {270                    // grammar_type key is set by the server when converting chat template grammars271                    std::string grammar_type = json_value(data, "grammar_type", std::string());272                    if (grammar_type == "tool_calls") {273                        params.sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, std::move(grammar_str)};274                    } else {275                        // explicit grammar from the user (API field "grammar")276                        params.sampling.grammar = {COMMON_GRAMMAR_TYPE_USER, std::move(grammar_str)};277                    }278                    SRV_DBG("Grammar (%s): %s\n", grammar_type.c_str(), common_grammar_value(params.sampling.grammar).c_str());279                }280            }281        }));282 283    add((new field_bool("grammar_lazy", params.sampling.grammar_lazy))284        ->set_desc("Whether to apply grammar constraints lazily, only when triggered (instead of at every step)"));285 286    //287    // Chat parser params288    //289 290    // TODO: change this to string field instead291    add((new field_json("chat_format"))292        ->set_desc("Chat format used internally by the server")293        ->set_handler([&](field_eval_context & ctx, const json & data) {294            ctx.params.chat_parser_params.format = static_cast<common_chat_format>(data.at("chat_format").get<int>());295            SRV_TRC("chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format));296        }));297 298    add((new field_str("reasoning_format"))299        ->set_desc("Reasoning format for chain-of-thought models")300        ->set_handler([&](field_eval_context & ctx, const json & data) {301            auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());302            ctx.params.chat_parser_params.reasoning_format = reasoning_format;303            ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);304        }));305 306    add((new field_str("generation_prompt"))307        ->set_desc("Generation prompt appended to the chat template output")308        ->set_handler([&](field_eval_context & ctx, const json & data) {309            std::string s = data.at("generation_prompt").get<std::string>();310            ctx.params.chat_parser_params.generation_prompt = s;311            ctx.params.sampling.generation_prompt = s;312        }));313 314    add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))315        ->set_desc("Whether to parse tool calls from the generated output"));316 317    add((new field_str("chat_parser"))318        ->set_desc("Chat parser configuration string")319        ->set_handler([&](field_eval_context & ctx, const json & data) {320            ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());321        }));322 323    add((new field_json("continue_final_message"))324        ->set_desc("Whether to continue the final message of the chat template")325        ->set_handler([&](field_eval_context & ctx, const json & data) {326            auto continuation = common_chat_continuation_parse(data.at("continue_final_message"));327            ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE;328        }));329 330    add((new field_bool("echo", params.chat_parser_params.echo))331        ->set_desc("Whether to echo the input tokens in the output"));332 333    //334    // Token-level fields (require vocab)335    //336 337    add((new field_json("preserved_tokens"))338        ->set_desc("List of token strings that must not be split during tokenization")339        ->set_handler([&](field_eval_context & ctx, const json & data) {340            GGML_ASSERT(ctx.vocab != nullptr);341            for (const auto & t : data.at("preserved_tokens")) {342                auto ids = common_tokenize(ctx.vocab, t.get<std::string>(), false, true);343                if (ids.size() == 1) {344                    ctx.params.sampling.preserved_tokens.insert(ids[0]);345                }346            }347        }));348 349    add((new field_json("grammar_triggers"))350        ->set_desc("List of strings or patterns that trigger grammar-constrained generation")351        ->set_handler([&](field_eval_context & ctx, const json & data) {352            GGML_ASSERT(ctx.vocab != nullptr);353            for (const auto & t : data.at("grammar_triggers")) {354                server_grammar_trigger ct(t);355                if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {356                    const auto & word = ct.value.value;357                    auto ids = common_tokenize(ctx.vocab, word, false, true);358                    if (ids.size() == 1) {359                        auto token = ids[0];360                        if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) {361                            throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);362                        }363                        common_grammar_trigger trigger;364                        trigger.type  = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;365                        trigger.value = word;366                        trigger.token = token;367                        ctx.params.sampling.grammar_triggers.push_back(std::move(trigger));368                    } else {369                        ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});370                    }371                } else {372                    ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value));373                }374            }375            if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) {376                throw std::runtime_error("Error: no triggers set for lazy grammar!");377            }378        }));379 380    add((new field_bool("reasoning_control", params.sampling.reasoning_control))381        ->set_desc("Create the budget sampler on demand so reasoning can be ended at runtime"));382 383    add((new field_num("reasoning_budget_tokens", params.sampling.reasoning_budget_tokens))384        ->set_hard_limits(-1, INT32_MAX)385        ->set_desc("Number of tokens in the reasoning budget (-1 = disabled)"));386 387    add((new field_str("reasoning_budget_start_tag"))388        ->set_desc("Token string marking the start of the reasoning budget section")389        ->set_handler([&](field_eval_context & ctx, const json & data) {390            GGML_ASSERT(ctx.vocab != nullptr);391            ctx.params.sampling.reasoning_budget_start = common_tokenize(ctx.vocab, data.at("reasoning_budget_start_tag").get<std::string>(), false, true);392        }));393 394    add((new field_json("reasoning_budget_end_tags"))395        ->add_alias("reasoning_budget_end_tag")396        ->set_desc("Token strings marking the end of the reasoning budget section; the first is forced when the budget expires")397        ->set_handler([&](field_eval_context & ctx, const json & data) {398            GGML_ASSERT(ctx.vocab != nullptr);399            ctx.params.sampling.reasoning_budget_end.clear();400            if (data.contains("reasoning_budget_end_tags")) {401                for (const auto & t : data.at("reasoning_budget_end_tags")) {402                    std::string tag = t.get<std::string>();403                    if (!tag.empty()) {404                        ctx.params.sampling.reasoning_budget_end.push_back(common_tokenize(ctx.vocab, tag, false, true));405                    }406                }407            } else if (data.contains("reasoning_budget_end_tag")) {408                std::string tag = data.at("reasoning_budget_end_tag").get<std::string>();409                if (!tag.empty()) {410                    ctx.params.sampling.reasoning_budget_end.push_back(common_tokenize(ctx.vocab, tag, false, true));411                }412            }413        }));414 415    add((new field_str("reasoning_budget_message"))416        ->set_desc("Message to prepend to the reasoning budget end tag when forcing it")417        ->set_handler([&](field_eval_context & ctx, const json & data) {418            GGML_ASSERT(ctx.vocab != nullptr);419            if (!ctx.params.sampling.reasoning_budget_end.empty()) {420                llama_tokens end_tag = ctx.params.sampling.reasoning_budget_end.front();421                std::string message = json_value(data, "reasoning_budget_message", std::string());422                if (!message.empty()) {423                    llama_tokens message_tokens = common_tokenize(ctx.vocab, message, false, true);424                    end_tag.insert(end_tag.begin(), message_tokens.begin(), message_tokens.end());425                }426                ctx.params.sampling.reasoning_budget_forced = std::move(end_tag);427            }428        }));429 430    add((new field_json("logit_bias"))431        ->set_desc("Modify the likelihood of specific tokens. Accepts an array of [token, bias] pairs or an object mapping token to bias. Use false as bias to ban a token")432        ->set_handler([&](field_eval_context & ctx, const json & data) {433            GGML_ASSERT(ctx.vocab != nullptr);434            ctx.params.sampling.logit_bias.clear();435            const auto & logit_bias = data.at("logit_bias");436            const int n_vocab = llama_vocab_n_tokens(ctx.vocab);437            auto parse_bias = [](const json & v, float & bias) -> bool {438                if (v.is_number())                        { bias = v.get<float>(); return true; }439                if (v.is_boolean() && !v.get<bool>())     { bias = -INFINITY;      return true; }440                return false;441            };442            if (logit_bias.is_array()) {443                for (const auto & el : logit_bias) {444                    if (!el.is_array() || el.size() != 2) continue;445                    float bias;446                    if (!parse_bias(el[1], bias)) continue;447                    if (el[0].is_number_integer()) {448                        llama_token tok = el[0].get<llama_token>();449                        if (tok >= 0 && tok < n_vocab) ctx.params.sampling.logit_bias.push_back({tok, bias});450                    } else if (el[0].is_string()) {451                        for (auto tok : common_tokenize(ctx.vocab, el[0].get<std::string>(), false))452                            ctx.params.sampling.logit_bias.push_back({tok, bias});453                    }454                }455            } else if (logit_bias.is_object()) {456                for (const auto & el : logit_bias.items()) {457                    float bias;458                    if (!parse_bias(el.value(), bias)) continue;459                    char * end;460                    llama_token tok = strtol(el.key().c_str(), &end, 10);461                    if (*end == 0) {462                        if (tok >= 0 && tok < n_vocab) ctx.params.sampling.logit_bias.push_back({tok, bias});463                    } else {464                        for (auto t : common_tokenize(ctx.vocab, el.key(), false))465                            ctx.params.sampling.logit_bias.push_back({t, bias});466                    }467                }468            }469        }));470 471    add((new field_bool("ignore_eos", params.sampling.ignore_eos))472        ->set_desc("Ignore the end-of-sequence token and continue generating")473        ->set_handler([&](field_eval_context & ctx, const json & data) {474            GGML_ASSERT(ctx.logit_bias_eog != nullptr);475            ctx.params.sampling.ignore_eos = data.at("ignore_eos").get<bool>();476            if (ctx.params.sampling.ignore_eos && ctx.logit_bias_eog) {477                ctx.params.sampling.logit_bias.insert(478                    ctx.params.sampling.logit_bias.end(),479                    ctx.logit_bias_eog->begin(), ctx.logit_bias_eog->end());480            }481        }));482 483    add((new field_json("stop"))484        ->set_desc("Specify stopping strings. Generation stops when one is produced, and the string is not included in the output")485        ->set_handler([&](field_eval_context & ctx, const json & data) {486            ctx.params.antiprompt.clear();487            const auto & stop = data.at("stop");488            if (stop.is_array()) {489                for (const auto & word : stop) {490                    if (!word.empty()) ctx.params.antiprompt.push_back(word);491                }492            } else if (stop.is_string()) {493                ctx.params.antiprompt.push_back(stop.get<std::string>());494            }495            // fall back to CLI defaults if the request provided no effective stop strings496            if (ctx.params.antiprompt.empty()) {497                ctx.params.antiprompt = params_base.antiprompt;498            }499        }));500 501    add((new field_json("samplers"))502        ->set_desc("The order in which samplers are applied. An array of sampler type names, or a single string of sampler chars")503        ->set_handler([&](field_eval_context & ctx, const json & data) {504            const auto & samplers = data.at("samplers");505            if (samplers.is_array()) {506                ctx.params.sampling.samplers = common_sampler_types_from_names(samplers);507            } else if (samplers.is_string()) {508                ctx.params.sampling.samplers = common_sampler_types_from_chars(samplers.get<std::string>());509            }510        }));511 512    return fields;513}514 515task_params eval_llama_cmpl_schema(516                const llama_vocab * vocab,517                const common_params & params_base,518                const std::vector<llama_logit_bias> & logit_bias_eog,519                const json & data) {520    task_params params;521 522    // Sampling parameter defaults are loaded from the global server context (but individual requests can still override them)523    params.sampling      = params_base.sampling;524    params.speculative   = params_base.speculative;525    params.n_keep        = params_base.n_keep;526    params.n_predict     = params_base.n_predict;527    params.n_cache_reuse = params_base.n_cache_reuse;528    params.cache_prompt  = params_base.cache_prompt;529    params.antiprompt    = params_base.antiprompt;530    params.sse_ping_interval = params_base.sse_ping_interval;531 532    // enabling this will output extra debug information in the HTTP responses from the server533    params.verbose       = params_base.verbosity > 9;534 535    params.chat_parser_params.reasoning_format = params_base.reasoning_format;536 537    // create context and schema538    field_eval_context ctx(params);539    ctx.vocab          = vocab;540    ctx.logit_bias_eog = &logit_bias_eog;541 542    auto schema = make_llama_cmpl_schema(params_base, params);543 544    // eval all fields in the schema545    for (const auto & f : schema) {546        f->eval(ctx, data);547    }548 549    // post-processing550    {551        // if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here552        auto reasoning_format = params.chat_parser_params.reasoning_format;553        params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);554    }555 556    // debugging557    {558        auto budget = params.sampling.reasoning_budget_tokens;559        SRV_DBG("reasoning budget: tokens=%d, generation_prompt='%s', start=%zu toks, end=%zu seqs, forced=%zu toks\n",560                budget, params.sampling.generation_prompt.c_str(),561                params.sampling.reasoning_budget_start.size(),562                params.sampling.reasoning_budget_end.size(),563                params.sampling.reasoning_budget_forced.size());564    }565 566    return params;567}568 569//570// eval() implementations571//572 573static void handle_with_catch(const char * name, std::function<void()> func) {574    try {575        func();576    } catch (const std::exception & e) {577        throw std::invalid_argument(string_format("Field '%s': %s", name, e.what()));578    }579}580 581// treat a null value as absent so clients can send null to request the server default582static bool has_value(const json & data, const char * n) {583    auto it = data.find(n);584    return it != data.end() && !it->is_null();585}586 587template <typename T>588void field_num<T>::eval(field_eval_context & ctx, const json & data) {589    for (const auto & n : name) {590        if (has_value(data, n)) {591            handle_with_catch(n, [&]() {592                if (custom_handler) {593                custom_handler(ctx, data);594                } else if (!is_hard_limit) {595                    val = std::max(min, std::min(max, data.at(n).template get<T>()));596                } else {597                    T tmp = data.at(n).template get<T>();598                    if (tmp < min || tmp > max) {599                        throw std::invalid_argument(std::string("Value must be between ") + std::to_string(min) + " <= value <= " + std::to_string(max) + ", but got " + std::to_string(tmp));600                    }601                    val = tmp;602                }603            });604            return;605        }606    }607}608 609void field_str::eval(field_eval_context & ctx, const json & data) {610    GGML_ASSERT(custom_handler);611    for (const auto & n : name) {612        if (has_value(data, n)) {613            handle_with_catch(n, [&]() {614                custom_handler(ctx, data);615            });616            return;617        }618    }619}620 621void field_bool::eval(field_eval_context & ctx, const json & data) {622    for (const auto & n : name) {623        if (has_value(data, n)) {624            handle_with_catch(n, [&]() {625                if (custom_handler) {626                    custom_handler(ctx, data);627                } else {628                    val = data.at(n).get<bool>();629                }630            });631            return;632        }633    }634}635 636void field_json::eval(field_eval_context & ctx, const json & data) {637    GGML_ASSERT(custom_handler);638    for (const auto & n : name) {639        if (has_value(data, n)) {640            handle_with_catch(n, [&]() {641                custom_handler(ctx, data);642            });643            return;644        }645    }646}647 648void field_nested::eval(field_eval_context & ctx, const json & data) {649    for (const auto & n : name) {650        if (data.contains(n) && data.at(n).is_object()) {651            for (auto & f : subfields) {652                f->eval(ctx, data.at(n));653            }654            return;655        }656    }657}658 659} // namespace server_schema660