Brunobkr/llama.cpp_AlgMor24_github

1#include "common.h"2#include "http.h"3#include "server-http.h"4#include "server-common.h"5#include "ui.h"6 7#include <cpp-httplib/httplib.h>8 9#include <functional>10#include <future>11#include <memory>12#include <string>13#include <thread>14 15//16// HTTP implementation using cpp-httplib17//18 19class server_http_context::Impl {20public:21    std::unique_ptr<httplib::Server> srv;22};23 24server_http_context::server_http_context()25    : pimpl(std::make_unique<Impl>())26{}27 28server_http_context::~server_http_context() = default;29 30static void log_server_request(const httplib::Request & req, const httplib::Response & res) {31    // skip logging requests that are regularly sent, to avoid log spam32    if (req.path == "/health"33        || req.path == "/v1/health"34        || req.path == "/models"35        || req.path == "/v1/models"36        || req.path == "/props"37        || req.path == "/metrics"38    ) {39        return;40    }41 42    // reminder: this function is not covered by httplib's exception handler; if someone does more complicated stuff, think about wrapping it in try-catch43 44    SRV_TRC("done request: %s %s %s %d\n", req.method.c_str(), req.path.c_str(), req.remote_addr.c_str(), res.status);45 46    SRV_DBG("request:  %s\n", req.body.c_str());47    SRV_DBG("response: %s\n", res.body.c_str());48}49 50// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port)51static bool origin_is_localhost(const std::string & origin) {52    try {53        const std::string host = common_http_parse_url(origin).host;54        return host == "localhost" || host == "127.0.0.1" || host == "::1";55    } catch (const std::exception &) {56        return false;57    }58}59 60// For Google Cloud Platform deployment compatibility61struct gcp_params {62    bool enabled;63    std::string path_health;64    std::string path_predict;65    int port;66 67    // Ref: https://docs.cloud.google.com/vertex-ai/docs/predictions/custom-container-requirements#aip-variables68    gcp_params() {69        enabled = getenv("AIP_MODE", "") == "PREDICTION";70        path_health = getenv("AIP_HEALTH_ROUTE", "", true); // default: using the route defined in server.cpp71        path_predict = getenv("AIP_PREDICT_ROUTE", "/predict", true);72        port = std::stoi(getenv("AIP_HTTP_PORT", "8080"));73    }74 75    static std::string getenv(const char * name, const std::string & default_value, bool ensure_leading_slash = false) {76        const auto * value = std::getenv(name);77        if (value == nullptr || value[0] == '\0') {78            return default_value;79        }80        std::string val = value;81        if (ensure_leading_slash && !val.empty() && val[0] != '/') {82            val.insert(val.begin(), '/');83        }84        return val;85    }86};87 88bool server_http_context::init(const common_params & params) {89    const gcp_params gcp;90 91    path_prefix = params.api_prefix;92    port = params.port;93    hostname = params.hostname;94 95    if (gcp.enabled) {96        SRV_TRC("Google Cloud Platform compat: health route = %s, predict route = %s, port = %d\n", gcp.path_health.c_str(), gcp.path_predict.c_str(), gcp.port);97 98        if (port != gcp.port) {99            SRV_WRN("Google Cloud Platform compat: overriding server port %d with AIP_HTTP_PORT %d\n", port, gcp.port);100        }101 102        port = gcp.port;103    }104 105    auto & srv = pimpl->srv;106 107#ifdef CPPHTTPLIB_OPENSSL_SUPPORT108    if (!params.ssl_file_key.empty() && !params.ssl_file_cert.empty()) {109        SRV_TRC("running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());110        srv = std::make_unique<httplib::SSLServer>(111            params.ssl_file_cert.c_str(), params.ssl_file_key.c_str()112        );113        is_ssl = true;114    } else {115        SRV_TRC("%s", "running without SSL\n");116        srv = std::make_unique<httplib::Server>();117    }118#else119    if (params.ssl_file_key != "" && params.ssl_file_cert != "") {120        SRV_ERR("%s", "the server is built without SSL support\n");121        return false;122    }123    srv.reset(new httplib::Server());124#endif125 126    srv->set_default_headers({{"Server", "llama.cpp"}});127    // srv->set_logger(log_server_request); // TODO @ngxson : this is too spamy, no very useful; improve it in the future128    srv->set_exception_handler([](const httplib::Request &, httplib::Response & res, const std::exception_ptr & ep) {129        // this is fail-safe; exceptions should already handled by `ex_wrapper`130 131        std::string message;132        try {133            std::rethrow_exception(ep);134        } catch (const std::exception & e) {135            message = e.what();136        } catch (...) {137            message = "Unknown Exception";138        }139 140        res.status = 500;141        res.set_content(message, "text/plain");142        SRV_ERR("got exception: %s\n", message.c_str());143    });144 145    srv->set_error_handler([](const httplib::Request &, httplib::Response & res) {146        if (res.status == 404) {147            res.set_content(148                safe_json_to_str(json {149                    {"error", {150                        {"message", "File Not Found"},151                        {"type", "not_found_error"},152                        {"code", 404}153                    }}154                }),155                "application/json; charset=utf-8"156            );157        }158        // for other error codes, we skip processing here because it's already done by res->error()159    });160 161    // set timeouts and change hostname and port162    srv->set_read_timeout (params.timeout_read);163    srv->set_write_timeout(params.timeout_write);164    srv->set_socket_options([reuse_port = params.reuse_port](const socket_t sock) {165        httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEADDR, 1);166        if (reuse_port) {167#ifdef SO_REUSEPORT168            httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEPORT, 1);169#else170            SRV_WRN("%s", "SO_REUSEPORT is not supported\n");171#endif172        }173    });174 175    if (params.api_keys.size() == 1) {176        const auto key = params.api_keys[0];177        const std::string substr = key.substr(std::max(static_cast<int>(key.length() - 4), 0));178        SRV_TRC("api_keys: ****%s\n", substr.c_str());179    } else if (params.api_keys.size() > 1) {180        SRV_TRC("api_keys: %zu keys loaded\n", params.api_keys.size());181    }182 183    //184    // Middlewares185    //186 187    // Frontend paths - all embedded UI assets188    static const std::unordered_set<std::string> frontend_paths = []() {189        std::unordered_set<std::string> paths { "/" };190        for (const llama_ui_asset & a : llama_ui_get_assets()) {191            paths.insert("/" + a.name);192        }193        return paths;194    }();195 196    // Public endpoints - API routes plus all embedded UI assets197    static const std::unordered_set<std::string> get_public_endpoints = []() {198        std::unordered_set<std::string> endpoints {199            "/health",200            "/v1/health",201            "/models",202            "/v1/models",203        };204        endpoints.insert(frontend_paths.begin(), frontend_paths.end());205        return endpoints;206    }();207 208    auto middleware_validate_api_key = [api_keys = params.api_keys](const httplib::Request & req, httplib::Response & res) {209        // If API key is not set, skip validation210        if (api_keys.empty()) {211            return true;212        }213 214        // If path is public or a UI asset, skip validation215        if (get_public_endpoints.count(req.path)) {216            return true;217        }218 219        // Check for API key in the Authorization header220        std::string req_api_key = req.get_header_value("Authorization");221        if (req_api_key.empty()) {222            // retry with anthropic header223            req_api_key = req.get_header_value("X-Api-Key");224        }225 226        // remove the "Bearer " prefix if needed227        static std::string prefix = "Bearer ";228        if (req_api_key.substr(0, prefix.size()) == prefix) {229            req_api_key = req_api_key.substr(prefix.size());230        }231 232        // validate the API key233        if (std::find(api_keys.begin(), api_keys.end(), req_api_key) != api_keys.end()) {234            return true; // API key is valid235        }236 237        // API key is invalid or not provided238        res.status = 401;239        res.set_content(240            safe_json_to_str(json {241                {"error", {242                    {"message", "Invalid API Key"},243                    {"type", "authentication_error"},244                    {"code", 401}245                }}246            }),247            "application/json; charset=utf-8"248        );249 250        SRV_WRN("%s", "unauthorized: Invalid API Key\n");251 252        return false;253    };254 255    auto middleware_server_state = [this](const httplib::Request & req, httplib::Response & res) {256        if (!is_ready.load()) {257            if (frontend_paths.count(req.path)) {258                return true; // frontend asset, allow it to load and show "loading"259            }260            // no endpoints are allowed to be accessed when the server is not ready261            // this is to prevent any data races or inconsistent states262            res.status = 503;263            res.set_content(264                safe_json_to_str(json {265                    {"error", {266                        {"message", "Loading model"},267                        {"type", "unavailable_error"},268                        {"code", 503}269                    }}270                }),271                "application/json; charset=utf-8"272            );273            return false;274        }275        return true;276    };277 278    // register server middlewares279    srv->set_pre_routing_handler([&params, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {280        if (params.cors_credentials && params.cors_origins == "*") {281            // special case: echo back the Origin header to allow any origin to access the server with credentials282            res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));283        } else if (params.cors_origins == "localhost") {284            // special case: only reflect the Origin header if it is a localhost origin285            std::string origin = req.get_header_value("Origin");286            if (!origin.empty() && origin_is_localhost(origin)) {287                res.set_header("Access-Control-Allow-Origin", origin);288            } else if (!origin.empty()) {289                SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str());290            }291        } else {292            res.set_header("Access-Control-Allow-Origin", params.cors_origins);293        }294        // If this is OPTIONS request, skip validation because browsers don't include Authorization header295        if (req.method == "OPTIONS") {296            res.set_header("Access-Control-Allow-Credentials", params.cors_credentials ? "true" : "false");297            res.set_header("Access-Control-Allow-Methods",     params.cors_methods);298            res.set_header("Access-Control-Allow-Headers",     params.cors_headers);299            res.set_content("", "text/html"); // blank response, no data300            return httplib::Server::HandlerResponse::Handled; // skip further processing301        }302        if (!middleware_server_state(req, res)) {303            return httplib::Server::HandlerResponse::Handled;304        }305        if (!middleware_validate_api_key(req, res)) {306            return httplib::Server::HandlerResponse::Handled;307        }308        return httplib::Server::HandlerResponse::Unhandled;309    });310 311    auto n_threads_http = params.n_threads_http;312    if (n_threads_http < 1) {313        // +4 threads for monitoring, health and some threads reserved for MCP and other tasks in the future314        n_threads_http = std::max(params.n_parallel + 4, static_cast<int32_t>(std::thread::hardware_concurrency() - 1));315    }316    SRV_TRC("using %d threads for HTTP server\n", n_threads_http);317    srv->new_task_queue = [n_threads_http] {318        // spawn n_threads_http fixed thread (always alive), while allow up to 1024 max possible additional threads319        // when n_threads_http is used, server will create new "dynamic" threads that will be destroyed after processing each request320        // ref: https://github.com/yhirose/cpp-httplib/pull/2368321        const auto max_threads = static_cast<size_t>(n_threads_http + 1024);322        return new httplib::ThreadPool(n_threads_http, max_threads);323    };324 325    //326    // Web UI setup327    //328 329    // Use new `params.ui` field (backed by old `params.webui` for compat)330    if (!params.ui) {331        SRV_INF("%s", "The UI is disabled\n");332        SRV_INF("%s", "Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable\n");333    } else {334        // register static assets routes335        if (!params.public_path.empty()) {336            // Set the base directory for serving static files337            if (const auto is_found = srv->set_mount_point(params.api_prefix + "/", params.public_path); !is_found) {338                SRV_ERR("static assets path not found: %s\n", params.public_path.c_str());339                return false;340            }341        } else {342#if defined(LLAMA_UI_HAS_ASSETS)343            static auto handle_gzip_header = [](const httplib::Request & req, httplib::Response & res) {344                if (!llama_ui_use_gzip()) {345                    // no gzip build, skip346                    return true;347                }348                if (req.get_header_value("Accept-Encoding").find("gzip") == std::string::npos) {349                    res.status = 415; // unsupported media type350                    res.set_content("Error: gzip is not supported by this browser", "text/plain");351                    return false;352                } else {353                    res.set_header("Content-Encoding", "gzip");354                }355                return true;356            };357 358            auto serve_asset_cached = [](const std::string & name, bool isolation) {359                return [name, isolation](const httplib::Request & req, httplib::Response & res) {360                    if (!handle_gzip_header(req, res)) {361                        return true; // returns error message362                    }363                    const llama_ui_asset * a = llama_ui_find_asset(name);364                    if (!a) { res.status = 404; return false; }365                    res.set_header("ETag", a->etag);366                    if (const std::string & inm = req.get_header_value("If-None-Match");367                        !inm.empty() && (inm == a->etag || inm == std::string("W/") + a->etag)) {368                        res.status = 304;369                        return false;370                    }371                    if (isolation) {372                        res.set_header("Cross-Origin-Embedder-Policy", "require-corp");373                        res.set_header("Cross-Origin-Opener-Policy",   "same-origin");374                    }375                    res.set_header("Cache-Control", "public, max-age=31536000, immutable");376                    res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());377                    return false;378                };379            };380 381            auto serve_asset_nocache = [](const std::string & name) {382                return [name](const httplib::Request & req, httplib::Response & res) {383                    if (!handle_gzip_header(req, res)) {384                        return true; // returns error message385                    }386                    const llama_ui_asset * a = llama_ui_find_asset(name);387                    if (!a) {388                        res.status = 404;389                        return false;390                    }391                    res.set_header("Cache-Control", "no-cache");392                    res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());393                    return false;394                };395            };396 397            // main index file398            srv->Get(params.api_prefix + "/",           serve_asset_cached("index.html", true));399            srv->Get(params.api_prefix + "/index.html", serve_asset_cached("index.html", true));400 401            // All remaining assets registered directly from the embedded asset table.402            // PWA revalidation files (sw.js, manifest, version.json) use no-cache;403            // everything else is immutable.404            static const std::unordered_set<std::string> no_cache_names = {405                "sw.js",406                "manifest.webmanifest",407                "_app/version.json",408                "build.json"409            };410 411            for (const auto & a : llama_ui_get_assets()) {412                if (a.name == "index.html") continue;  // served at "/" and "/index.html" above413                if (no_cache_names.count(a.name)) {414                    SRV_DBG("serve nocache for %s\n", a.name.c_str());415                    srv->Get(params.api_prefix + "/" + a.name, serve_asset_nocache(a.name));416                } else {417                    srv->Get(params.api_prefix + "/" + a.name, serve_asset_cached(a.name, false));418                }419            }420 421#endif422        }423    }424    return true;425}426 427bool server_http_context::start() {428    // Bind and listen429 430    const auto & srv = pimpl->srv;431    auto was_bound = false;432    auto is_sock = false;433    if (string_ends_with(std::string(hostname), ".sock")) {434        is_sock = true;435        SRV_TRC("%s", "setting address family to AF_UNIX\n");436        srv->set_address_family(AF_UNIX);437        // bind_to_port requires a second arg, any value other than 0 should438        // simply get ignored439        was_bound = srv->bind_to_port(hostname, 8080);440    } else {441        SRV_TRC("%s", "binding port with default address family\n");442        // bind HTTP listen port443        if (port == 0) {444            const auto bound_port = srv->bind_to_any_port(hostname);445            was_bound = (bound_port >= 0);446            if (was_bound) {447                port = bound_port;448            }449        } else {450            was_bound = srv->bind_to_port(hostname, port);451        }452    }453 454    if (!was_bound) {455        SRV_ERR("couldn't bind HTTP server socket, hostname: %s, port: %d\n", hostname.c_str(), port);456        return false;457    }458 459    // run the HTTP server in a thread460    thread = std::thread([this] { pimpl->srv->listen_after_bind(); });461    srv->wait_until_ready();462 463    listening_address = is_sock ? string_format("unix://%s", hostname.c_str())464                                : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(hostname).c_str(), port);465    return true;466}467 468void server_http_context::stop() const {469    if (pimpl->srv) {470        pimpl->srv->stop();471    }472}473 474static void set_headers(httplib::Response & res, const std::map<std::string, std::string> & headers) {475    for (const auto & [key, value] : headers) {476        res.set_header(key, value);477    }478}479 480// percent-decode a path component (%XX). path params arrive raw from httplib, unlike query481// params, so a conv id like "conv::model" sent as "conv%3A%3Amodel" must be decoded here to482// match the value the client put in the X-Conversation-Id header483static std::string decode_path_component(const std::string & in) {484    std::string out;485    out.reserve(in.size());486    for (size_t i = 0; i < in.size(); i++) {487        if (in[i] == '%' && i + 2 < in.size()) {488            auto hex = [](char c) -> int {489                if (c >= '0' && c <= '9') return c - '0';490                if (c >= 'a' && c <= 'f') return c - 'a' + 10;491                if (c >= 'A' && c <= 'F') return c - 'A' + 10;492                return -1;493            };494            int hi = hex(in[i + 1]);495            int lo = hex(in[i + 2]);496            if (hi >= 0 && lo >= 0) {497                out.push_back(char((hi << 4) | lo));498                i += 2;499                continue;500            }501        }502        out.push_back(in[i]);503    }504    return out;505}506 507static std::map<std::string, std::string> get_params(const httplib::Request & req) {508    std::map<std::string, std::string> params;509    for (const auto & [key, value] : req.params) {510        params[key] = value;511    }512    for (const auto & [key, value] : req.path_params) {513        params[key] = decode_path_component(value);514    }515    return params;516}517 518static std::map<std::string, std::string> get_headers(const httplib::Request & req) {519    std::map<std::string, std::string> headers;520    for (const auto & [key, value] : req.headers) {521        headers[key] = value;522    }523    return headers;524}525 526static std::string build_query_string(const httplib::Request & req) {527    std::string qs;528    for (const auto & [key, value] : req.params) {529        if (!qs.empty()) {530            qs += '&';531        }532        qs += httplib::encode_query_component(key) + "=" + httplib::encode_query_component(value);533    }534    return qs;535}536 537// using unique_ptr for request to allow safe capturing in lambdas538using server_http_req_ptr = std::unique_ptr<server_http_req>;539 540static void process_handler_response(server_http_req_ptr && request, server_http_res_ptr & response, httplib::Response & res) {541    if (response->is_stream()) {542        res.status = response->status;543        // Tell Nginx to not buffer any streamed response544        response->headers["X-Accel-Buffering"] = "no";545        set_headers(res, response->headers);546        const std::string content_type = response->content_type;547        // convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it548        std::shared_ptr<server_http_req> q_ptr = std::move(request);549        std::shared_ptr<server_http_res> r_ptr = std::move(response);550 551        const auto chunked_content_provider = [response = r_ptr](size_t, httplib::DataSink & sink) -> bool {552            std::string chunk;553            const bool has_next = response->next(chunk);554            if (!chunk.empty()) {555                if (!sink.write(chunk.data(), chunk.size())) {556                    return false;557                }558                SRV_DBG("http: streamed chunk: %s\n", chunk.c_str());559            }560            if (!has_next) {561                sink.done();562                SRV_DBG("%s", "http: stream ended\n");563            }564            return has_next;565        };566        const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable {567            response->on_complete();568            response.reset();569            request.reset();570        };571        res.set_chunked_content_provider(content_type, chunked_content_provider, on_complete);572    } else {573        res.status = response->status;574        set_headers(res, response->headers);575        res.set_content(response->data, response->content_type);576        response->on_complete();577    }578}579 580void server_http_context::get(const std::string & path, const server_http_context::handler_t & handler) const {581    handlers.emplace(path, handler);582    pimpl->srv->Get(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {583        server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{584            get_params(req),585            get_headers(req),586            req.path,587            build_query_string(req),588            req.body,589            {},590            req.is_connection_closed591        });592        server_http_res_ptr response = handler(*request);593        process_handler_response(std::move(request), response, res);594    });595}596 597void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const {598    handlers.emplace(path, handler);599    pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {600        std::string body = req.body;601        std::map<std::string, uploaded_file> files;602 603        if (req.is_multipart_form_data()) {604            // translate text fields to a JSON object and use it as the body605            json form_json = json::object();606            for (const auto & [key, field] : req.form.fields) {607                if (form_json.contains(key)) {608                    // if the key already exists, convert it to an array609                    if (!form_json[key].is_array()) {610                        json existing_value = form_json[key];611                        form_json[key] = json::array({existing_value});612                    }613                    form_json[key].push_back(field.content);614                } else {615                    form_json[key] = field.content;616                }617            }618            body = form_json.dump();619 620            // populate files from multipart form621            for (const auto & [key, file] : req.form.files) {622                files[key] = uploaded_file{623                    raw_buffer(file.content.begin(), file.content.end()),624                    file.filename,625                    file.content_type,626                };627            }628        }629 630        server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{631            get_params(req),632            get_headers(req),633            req.path,634            build_query_string(req),635            body,636            std::move(files),637            req.is_connection_closed638        });639        server_http_res_ptr response = handler(*request);640        process_handler_response(std::move(request), response, res);641    });642}643 644void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const {645    handlers.emplace(path, handler);646    pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {647        server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{648            get_params(req),649            get_headers(req),650            req.path,651            build_query_string(req),652            req.body,653            {},654            req.is_connection_closed655        });656        server_http_res_ptr response = handler(*request);657        process_handler_response(std::move(request), response, res);658    });659}660 661//662// Vertex AI Prediction protocol (AIP_PREDICT_ROUTE)663// https://cloud.google.com/vertex-ai/docs/predictions/custom-container-requirements664//665 666// Derives the camelCase @requestFormat alias for a registered path.667// e.g. "/v1/chat/completions" -> "chatCompletions", "/apply-template" -> "applyTemplate"668static std::string path_to_gcp_format(const std::string & path) {669    std::string s = path;670    if (s.size() > 3 && s[0] == '/' && s[1] == 'v' && s[2] == '1') {671        s = s.substr(3);672    }673    if (!s.empty() && s[0] == '/') {674        s = s.substr(1);675    }676    std::string result;677    bool cap = false;678    for (unsigned char c : s) {679        if (c == ':') break; // stop before path parameters680        if (c == '/' || c == '-' || c == '_') {681            cap = true;682        } else {683            result += static_cast<char>(cap ? std::toupper(c) : c);684            cap = false;685        }686    }687    return result;688}689 690static json parse_gcp_predict_response(const server_http_res_ptr & res) {691    if (res == nullptr) {692        throw std::runtime_error("empty response from internal handler");693    }694    if (res->is_stream()) {695        throw std::invalid_argument("predict route does not support streaming responses");696    }697    if (res->data.empty()) {698        return nullptr;699    }700    try {701        return json::parse(res->data);702    } catch (...) {703        return res->data;704    }705}706 707void server_http_context::register_gcp_compat() const {708    const gcp_params gcp;709 710    if (!gcp.enabled) {711        // do nothing712        return;713    }714 715    if (handlers.count(gcp.path_predict)) {716        SRV_ERR("AIP_PREDICT_ROUTE=%s conflicts with an existing llama-server route\n", gcp.path_predict.c_str());717        exit(1);718    }719 720    // camelCase alias -> canonical path (first registration wins on collision)721    // e.g. "chatCompletions" -> "/v1/chat/completions"722    std::unordered_map<std::string, std::string> alias_to_path;723    for (const auto & [path, _] : handlers) {724        alias_to_path.emplace(path_to_gcp_format(path), path);725    }726 727    if (!gcp.path_health.empty()) {728        const auto health_handler = handlers.find("/health");729        GGML_ASSERT(health_handler != handlers.end());730        get(gcp.path_health, health_handler->second);731    }732 733    post(gcp.path_predict, [this, alias_to_path = std::move(alias_to_path)](const server_http_req & req) -> server_http_res_ptr {734        static const auto build_error = [](const std::string & message, error_type type) -> json {735            return json {{"error", format_error_response(message, type)}};736        };737 738        json data;739        try {740            data = json::parse(req.body);741        } catch (const std::exception & e) {742            auto res = std::make_unique<server_http_res>();743            res->status = 400;744            res->data = safe_json_to_str({{"error", format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)}});745            return res;746        }747        if (!data.is_object()) {748            auto res = std::make_unique<server_http_res>();749            res->status = 400;750            res->data = safe_json_to_str({{"error", format_error_response("request body must be a JSON object", ERROR_TYPE_INVALID_REQUEST)}});751            return res;752        }753        if (!data.contains("instances") || !data.at("instances").is_array()) {754            auto res = std::make_unique<server_http_res>();755            res->status = 400;756            res->data = safe_json_to_str({{"error", format_error_response("request body must include an array field named instances", ERROR_TYPE_INVALID_REQUEST)}});757            return res;758        }759 760        const json & instances = data.at("instances");761        static const size_t MAX_INSTANCES = 128;762        if (instances.size() > MAX_INSTANCES) {763            auto res = std::make_unique<server_http_res>();764            res->status = 400;765            res->data = safe_json_to_str({{"error", format_error_response("instances array exceeds maximum size of " + std::to_string(MAX_INSTANCES), ERROR_TYPE_INVALID_REQUEST)}});766            return res;767        }768 769        std::vector<std::future<json>> futures;770        futures.reserve(instances.size());771 772        for (const auto & instance : instances) {773            futures.push_back(std::async(std::launch::async, [this, &req, &alias_to_path, instance]() -> json {774                if (!instance.is_object()) {775                    return build_error("each instance must be a JSON object", ERROR_TYPE_INVALID_REQUEST);776                }777                if (!instance.contains("@requestFormat") || !instance.at("@requestFormat").is_string()) {778                    return build_error("each instance must include a string @requestFormat", ERROR_TYPE_INVALID_REQUEST);779                }780 781                try {782                    json payload = instance;783                    const std::string format = payload.at("@requestFormat").get<std::string>();784                    payload.erase("@requestFormat");785 786                    if (payload.contains("stream")) {787                        SRV_WRN("%s", "ignoring client-provided stream field in instance, streaming is not supported in predict route\n");788                        payload["stream"] = false;789                    }790 791                    // accept both camelCase aliases (e.g. "chatCompletions") and direct paths792                    std::string dispatch_path;793                    auto it_alias = alias_to_path.find(format);794                    if (it_alias != alias_to_path.end()) {795                        dispatch_path = it_alias->second;796                    } else if (handlers.count(format)) {797                        dispatch_path = format;798                    } else {799                        return build_error("no handler registered for @requestFormat: " + format, ERROR_TYPE_INVALID_REQUEST);800                    }801 802                    const server_http_req internal_req {803                        req.params,804                        req.headers,805                        path_prefix + dispatch_path,806                        req.query_string,807                        payload.dump(),808                        {},809                        req.should_stop,810                    };811 812                    server_http_res_ptr internal_res = handlers.at(dispatch_path)(internal_req);813                    return parse_gcp_predict_response(internal_res);814                } catch (const std::invalid_argument & e) {815                    return build_error(e.what(), ERROR_TYPE_INVALID_REQUEST);816                } catch (const std::exception & e) {817                    return build_error(e.what(), ERROR_TYPE_SERVER);818                } catch (...) {819                    return build_error("unknown error", ERROR_TYPE_SERVER);820                }821            }));822        }823 824        json predictions = json::array();825        for (auto & future : futures) {826            predictions.push_back(future.get());827        }828 829        auto res = std::make_unique<server_http_res>();830        res->data = safe_json_to_str({{"predictions", predictions}});831        return res;832    });833}834