echodict/llama.cpp

1#include "arg.h"2#include "preset.h"3#include "peg-parser.h"4#include "log.h"5#include "download.h"6 7#include <fstream>8#include <sstream>9#include <filesystem>10 11static std::string rm_leading_dashes(const std::string & str) {12    size_t pos = 0;13    while (pos < str.size() && str[pos] == '-') {14        ++pos;15    }16    return str.substr(pos);17}18 19// only allow a subset of args for remote presets for security reasons20// do not add more args unless absolutely necessary21// args that output to files are strictly prohibited22static std::set<std::string> get_remote_preset_whitelist(const std::map<std::string, common_arg> & key_to_opt) {23    static const std::set<std::string> allowed_options = {24        "model-url",25        "hf-repo",26        "hf-repo-draft",27        "hf-repo-v", // vocoder28        "hf-file-v", // vocoder29        "mmproj-url",30        "pooling",31        "jinja",32        "batch-size",33        "ubatch-size",34        "cache-reuse",35        "chat-template-kwargs",36        "mmap",37        // note: sampling params are automatically allowed by default38        // negated args will be added automatically if the positive arg is specified above39    };40 41    std::set<std::string> allowed_keys;42 43    for (const auto & it : key_to_opt) {44        const std::string & key = it.first;45        const common_arg & opt = it.second;46        if (allowed_options.find(key) != allowed_options.end() || opt.is_sparam) {47            allowed_keys.insert(key);48            // also add variant keys (args without leading dashes and env vars)49            for (const auto & arg : opt.get_args()) {50                allowed_keys.insert(rm_leading_dashes(arg));51            }52            for (const auto & env : opt.get_env()) {53                allowed_keys.insert(env);54            }55        }56    }57 58    return allowed_keys;59}60 61std::vector<std::string> common_preset::to_args(const std::string & bin_path) const {62    std::vector<std::string> args;63 64    if (!bin_path.empty()) {65        args.push_back(bin_path);66    }67 68    for (const auto & [opt, value] : options) {69        if (opt.is_preset_only) {70            continue; // skip preset-only options (they are not CLI args)71        }72 73        // use the last arg as the main arg (i.e. --long-form)74        args.push_back(opt.args.back());75 76        // handle value(s)77        if (opt.value_hint == nullptr && opt.value_hint_2 == nullptr) {78            // flag option, no value79            if (common_arg_utils::is_falsey(value)) {80                // use negative arg if available81                if (!opt.args_neg.empty()) {82                    args.back() = opt.args_neg.back();83                } else {84                    // otherwise, skip the flag85                    // TODO: maybe throw an error instead?86                    args.pop_back();87                }88            }89        }90        if (opt.value_hint != nullptr) {91            // single value92            args.push_back(value);93        }94        if (opt.value_hint != nullptr && opt.value_hint_2 != nullptr) {95            throw std::runtime_error(string_format(96                "common_preset::to_args(): option '%s' has two values, which is not supported yet",97                opt.args.back()98            ));99        }100    }101 102    return args;103}104 105std::string common_preset::to_ini() const {106    std::ostringstream ss;107 108    ss << "[" << name << "]\n";109    for (const auto & [opt, value] : options) {110        auto espaced_value = value;111        string_replace_all(espaced_value, "\n", "\\\n");112        ss << rm_leading_dashes(opt.args.back()) << " = ";113        ss << espaced_value << "\n";114    }115    ss << "\n";116 117    return ss.str();118}119 120void common_preset::set_option(const common_preset_context & ctx, const std::string & env, const std::string & value) {121    // try if option exists, update it122    for (auto & [opt, val] : options) {123        if (opt.env && env == opt.env) {124            val = value;125            return;126        }127    }128    // if option does not exist, we need to add it129    if (ctx.key_to_opt.find(env) == ctx.key_to_opt.end()) {130        throw std::runtime_error(string_format(131            "%s: option with env '%s' not found in ctx_params",132            __func__, env.c_str()133        ));134    }135    options[ctx.key_to_opt.at(env)] = value;136}137 138void common_preset::unset_option(const std::string & env) {139    for (auto it = options.begin(); it != options.end(); ) {140        const common_arg & opt = it->first;141        if (opt.env && env == opt.env) {142            it = options.erase(it);143            return;144        } else {145            ++it;146        }147    }148}149 150bool common_preset::get_option(const std::string & env, std::string & value) const {151    for (const auto & [opt, val] : options) {152        if (opt.env && env == opt.env) {153            value = val;154            return true;155        }156    }157    return false;158}159 160void common_preset::merge(const common_preset & other) {161    for (const auto & [opt, val] : other.options) {162        options[opt] = val; // overwrite existing options163    }164}165 166void common_preset::apply_to_params(common_params & params) const {167    for (const auto & [opt, val] : options) {168        // apply each option to params169        if (opt.handler_string) {170            opt.handler_string(params, val);171        } else if (opt.handler_int) {172            opt.handler_int(params, std::stoi(val));173        } else if (opt.handler_bool) {174            opt.handler_bool(params, common_arg_utils::is_truthy(val));175        } else if (opt.handler_str_str) {176            // not supported yet177            throw std::runtime_error(string_format(178                "%s: option with two values is not supported yet",179                __func__180            ));181        } else if (opt.handler_void) {182            opt.handler_void(params);183        } else {184            GGML_ABORT("unknown handler type");185        }186    }187}188 189static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_file(const std::string & path) {190    std::map<std::string, std::map<std::string, std::string>> parsed;191 192    if (!std::filesystem::exists(path)) {193        throw std::runtime_error("preset file does not exist: " + path);194    }195 196    std::ifstream file(path);197    if (!file.good()) {198        throw std::runtime_error("failed to open server preset file: " + path);199    }200 201    std::string contents((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());202 203    static const auto parser = build_peg_parser([](auto & p) {204        // newline ::= "\r\n" / "\n" / "\r"205        auto newline = p.rule("newline", p.literal("\r\n") | p.literal("\n") | p.literal("\r"));206 207        // ws ::= [ \t]*208        auto ws = p.rule("ws", p.chars("[ \t]", 0, -1));209 210        // comment ::= [;#] (!newline .)*211        auto comment = p.rule("comment", p.chars("[;#]", 1, 1) + p.zero_or_more(p.negate(newline) + p.any()));212 213        // eol ::= ws comment? (newline / EOF)214        auto eol = p.rule("eol", ws + p.optional(comment) + (newline | p.end()));215 216        // ident ::= [a-zA-Z_] [a-zA-Z0-9_.-]*217        auto ident = p.rule("ident", p.chars("[a-zA-Z_]", 1, 1) + p.chars("[a-zA-Z0-9_.-]", 0, -1));218 219        // value ::= (!eol-start .)*220        auto eol_start = p.rule("eol-start", ws + (p.chars("[;#]", 1, 1) | newline | p.end()));221        auto value = p.rule("value", p.zero_or_more(p.negate(eol_start) + p.any()));222 223        // header-line ::= "[" ws ident ws "]" eol224        auto header_line = p.rule("header-line", "[" + ws + p.tag("section-name", p.chars("[^]]")) + ws + "]" + eol);225 226        // kv-line ::= ident ws "=" ws value eol227        auto kv_line = p.rule("kv-line", p.tag("key", ident) + ws + "=" + ws + p.tag("value", value) + eol);228 229        // comment-line ::= ws comment (newline / EOF)230        auto comment_line = p.rule("comment-line", ws + comment + (newline | p.end()));231 232        // blank-line ::= ws (newline / EOF)233        auto blank_line = p.rule("blank-line", ws + (newline | p.end()));234 235        // line ::= header-line / kv-line / comment-line / blank-line236        auto line = p.rule("line", header_line | kv_line | comment_line | blank_line);237 238        // ini ::= line* EOF239        auto ini = p.rule("ini", p.zero_or_more(line) + p.end());240 241        return ini;242    });243 244    common_peg_parse_context ctx(contents);245    const auto result = parser.parse(ctx);246    if (!result.success()) {247        throw std::runtime_error("failed to parse server config file: " + path);248    }249 250    std::string current_section = COMMON_PRESET_DEFAULT_NAME;251    std::string current_key;252 253    ctx.ast.visit(result, [&](const auto & node) {254        if (node.tag == "section-name") {255            const std::string section = std::string(node.text);256            current_section = section;257            parsed[current_section] = {};258        } else if (node.tag == "key") {259            const std::string key = std::string(node.text);260            current_key = key;261        } else if (node.tag == "value" && !current_key.empty() && !current_section.empty()) {262            parsed[current_section][current_key] = std::string(node.text);263            current_key.clear();264        }265    });266 267    return parsed;268}269 270static std::map<std::string, common_arg> get_map_key_opt(common_params_context & ctx_params) {271    std::map<std::string, common_arg> mapping;272    for (const auto & opt : ctx_params.options) {273        for (const auto & env : opt.get_env()) {274            mapping[env] = opt;275        }276        for (const auto & arg : opt.get_args()) {277            mapping[rm_leading_dashes(arg)] = opt;278        }279    }280    return mapping;281}282 283static bool is_bool_arg(const common_arg & arg) {284    return !arg.args_neg.empty();285}286 287static std::string parse_bool_arg(const common_arg & arg, const std::string & key, const std::string & value) {288    // if this is a negated arg, we need to reverse the value289    for (const auto & neg_arg : arg.args_neg) {290        if (rm_leading_dashes(neg_arg) == key) {291            return common_arg_utils::is_truthy(value) ? "false" : "true";292        }293    }294    // otherwise, not negated295    return value;296}297 298common_preset_context::common_preset_context(llama_example ex, bool only_remote_allowed)299        : ctx_params(common_params_parser_init(default_params, ex)) {300    common_params_add_preset_options(ctx_params.options);301    key_to_opt = get_map_key_opt(ctx_params);302 303    // setup allowed keys if only_remote_allowed is true304    if (only_remote_allowed) {305        filter_allowed_keys = true;306        allowed_keys = get_remote_preset_whitelist(key_to_opt);307    }308}309 310common_presets common_preset_context::load_from_ini(const std::string & path, common_preset & global) const {311    common_presets out;312    auto ini_data = parse_ini_from_file(path);313 314    for (auto section : ini_data) {315        common_preset preset;316        if (section.first.empty()) {317            preset.name = COMMON_PRESET_DEFAULT_NAME;318        } else {319            preset.name = section.first;320        }321        LOG_DBG("loading preset: %s\n", preset.name.c_str());322        for (const auto & [key, value] : section.second) {323            if (key == "version") {324                // skip version key (reserved for future use)325                continue;326            }327 328            LOG_DBG("option: %s = %s\n", key.c_str(), value.c_str());329            if (filter_allowed_keys && allowed_keys.find(key) == allowed_keys.end()) {330                throw std::runtime_error(string_format(331                    "option '%s' is not allowed in remote presets",332                    key.c_str()333                ));334            }335            if (key_to_opt.find(key) != key_to_opt.end()) {336                const auto & opt = key_to_opt.at(key);337                if (is_bool_arg(opt)) {338                    preset.options[opt] = parse_bool_arg(opt, key, value);339                } else {340                    preset.options[opt] = value;341                }342                LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());343            } else {344                throw std::runtime_error(string_format(345                    "option '%s' not recognized in preset '%s'",346                    key.c_str(), preset.name.c_str()347                ));348            }349        }350 351        if (preset.name == "*") {352            // handle global preset353            global = preset;354        } else {355            out[preset.name] = preset;356        }357    }358 359    return out;360}361 362common_presets common_preset_context::load_from_cache() const {363    common_presets out;364 365    auto cached_models = common_list_cached_models();366    for (const auto & model : cached_models) {367        common_preset preset;368        preset.name = model.to_string();369        preset.set_option(*this, "LLAMA_ARG_HF_REPO", model.to_string());370        out[preset.name] = preset;371    }372 373    return out;374}375 376struct local_model {377    std::string name;378    std::string path;379    std::string path_mmproj;380};381 382common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const {383    if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {384        throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str()));385    }386 387    std::vector<local_model> models;388    auto scan_subdir = [&models](const std::string & subdir_path, const std::string & name) {389        auto files = fs_list(subdir_path, false);390        common_file_info model_file;391        common_file_info first_shard_file;392        common_file_info mmproj_file;393        for (const auto & file : files) {394            if (string_ends_with(file.name, ".gguf")) {395                if (file.name.find("mmproj") != std::string::npos) {396                    mmproj_file = file;397                } else if (file.name.find("-00001-of-") != std::string::npos) {398                    first_shard_file = file;399                } else {400                    model_file = file;401                }402            }403        }404        // single file model405        local_model model{406            /* name        */ name,407            /* path        */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path,408            /* path_mmproj */ mmproj_file.path // can be empty409        };410        if (!model.path.empty()) {411            models.push_back(model);412        }413    };414 415    auto files = fs_list(models_dir, true);416    for (const auto & file : files) {417        if (file.is_dir) {418            scan_subdir(file.path, file.name);419        } else if (string_ends_with(file.name, ".gguf")) {420            // single file model421            std::string name = file.name;422            string_replace_all(name, ".gguf", "");423            local_model model{424                /* name        */ name,425                /* path        */ file.path,426                /* path_mmproj */ ""427            };428            models.push_back(model);429        }430    }431 432    // convert local models to presets433    common_presets out;434    for (const auto & model : models) {435        common_preset preset;436        preset.name = model.name;437        preset.set_option(*this, "LLAMA_ARG_MODEL", model.path);438        if (!model.path_mmproj.empty()) {439            preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj);440        }441        out[preset.name] = preset;442    }443 444    return out;445}446 447common_preset common_preset_context::load_from_args(int argc, char ** argv) const {448    common_preset preset;449    preset.name = COMMON_PRESET_DEFAULT_NAME;450 451    bool ok = common_params_to_map(argc, argv, ctx_params.ex, preset.options);452    if (!ok) {453        throw std::runtime_error("failed to parse CLI arguments into preset");454    }455 456    return preset;457}458 459common_presets common_preset_context::cascade(const common_presets & base, const common_presets & added) const {460    common_presets out = base; // copy461    for (const auto & [name, preset_added] : added) {462        if (out.find(name) != out.end()) {463            // if exists, merge464            common_preset & target = out[name];465            target.merge(preset_added);466        } else {467            // otherwise, add directly468            out[name] = preset_added;469        }470    }471    return out;472}473 474common_presets common_preset_context::cascade(const common_preset & base, const common_presets & presets) const {475    common_presets out;476    for (const auto & [name, preset] : presets) {477        common_preset tmp = base; // copy478        tmp.name = name;479        tmp.merge(preset);480        out[name] = std::move(tmp);481    }482    return out;483}484