Brunobkr/llama.cpp_AlgMor24_github

1#include "arg.h"2#include "preset.h"3#include "peg-parser.h"4#include "log.h"5#include "download.h"6 7#include <fstream>8#include <sstream>9#include <filesystem>10#include <regex>11 12static std::string rm_leading_dashes(const std::string & str) {13    size_t pos = 0;14    while (pos < str.size() && str[pos] == '-') {15        ++pos;16    }17    return str.substr(pos);18}19 20static std::string canonical_tag(const std::string & tag) {21    static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase);22    std::smatch m;23    if (std::regex_search(tag, m, re_tag)) {24        std::string canon = m[1].str();25        for (char & c : canon) {26            c = (char) std::toupper((unsigned char) c);27        }28        return canon;29    }30    std::string upper = tag;31    for (char & c : upper) {32        c = (char) std::toupper((unsigned char) c);33    }34    return upper;35}36 37std::vector<std::string> common_preset::to_args(const std::string & bin_path) const {38    std::vector<std::string> args;39 40    if (!bin_path.empty()) {41        args.push_back(bin_path);42    }43 44    for (const auto & [opt, value] : options) {45        if (opt.is_preset_only) {46            continue; // skip preset-only options (they are not CLI args)47        }48 49        // use the last arg as the main arg (i.e. --long-form)50        args.push_back(opt.args.back());51 52        // handle value(s)53        if (opt.value_hint == nullptr && opt.value_hint_2 == nullptr) {54            // flag option, no value55            if (common_arg_utils::is_falsey(value)) {56                // use negative arg if available57                if (!opt.args_neg.empty()) {58                    args.back() = opt.args_neg.back();59                } else {60                    // otherwise, skip the flag61                    // TODO: maybe throw an error instead?62                    args.pop_back();63                }64            }65        }66        if (opt.value_hint != nullptr) {67            // single value68            args.push_back(value);69        }70        if (opt.value_hint != nullptr && opt.value_hint_2 != nullptr) {71            throw std::runtime_error(string_format(72                "common_preset::to_args(): option '%s' has two values, which is not supported yet",73                opt.args.back()74            ));75        }76    }77 78    return args;79}80 81std::string common_preset::to_ini() const {82    std::ostringstream ss;83 84    ss << "[" << name << "]\n";85    for (const auto & [opt, value] : options) {86        auto espaced_value = value;87        string_replace_all(espaced_value, "\n", "\\\n");88        ss << rm_leading_dashes(opt.args.back()) << " = ";89        ss << espaced_value << "\n";90    }91    ss << "\n";92 93    return ss.str();94}95 96void common_preset::set_option(const common_preset_context & ctx, const std::string & env, const std::string & value) {97    // try if option exists, update it98    for (auto & [opt, val] : options) {99        if (opt.env && env == opt.env) {100            val = value;101            return;102        }103    }104    // if option does not exist, we need to add it105    if (ctx.key_to_opt.find(env) == ctx.key_to_opt.end()) {106        throw std::runtime_error(string_format(107            "%s: option with env '%s' not found in ctx_params",108            __func__, env.c_str()109        ));110    }111    options[ctx.key_to_opt.at(env)] = value;112}113 114void common_preset::unset_option(const std::string & env) {115    for (auto it = options.begin(); it != options.end(); ) {116        const common_arg & opt = it->first;117        if (opt.env && env == opt.env) {118            it = options.erase(it);119            return;120        } else {121            ++it;122        }123    }124}125 126bool common_preset::get_option(const std::string & env, std::string & value) const {127    for (const auto & [opt, val] : options) {128        if (opt.env && env == opt.env) {129            value = val;130            return true;131        }132    }133    return false;134}135 136void common_preset::merge(const common_preset & other) {137    for (const auto & [opt, val] : other.options) {138        options[opt] = val; // overwrite existing options139    }140}141 142void common_preset::apply_to_params(common_params & params, const std::set<std::string> & handled_keys) const {143    for (const auto & [opt, val] : options) {144        if (!handled_keys.empty()) {145            if (!opt.env || handled_keys.find(opt.env) == handled_keys.end()) {146                continue;147            }148        }149        // apply each option to params150        if (opt.handler_string) {151            opt.handler_string(params, val);152        } else if (opt.handler_int) {153            opt.handler_int(params, std::stoi(val));154        } else if (opt.handler_bool) {155            opt.handler_bool(params, common_arg_utils::is_truthy(val));156        } else if (opt.handler_str_str) {157            // not supported yet158            throw std::runtime_error(string_format(159                "%s: option with two values is not supported yet",160                __func__161            ));162        } else if (opt.handler_void) {163            opt.handler_void(params);164        } else {165            GGML_ABORT("unknown handler type");166        }167    }168}169 170static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_file(const std::string & path) {171    std::map<std::string, std::map<std::string, std::string>> parsed;172 173    if (!std::filesystem::exists(path)) {174        throw std::runtime_error("preset file does not exist: " + path);175    }176 177    std::ifstream file(path);178    if (!file.good()) {179        throw std::runtime_error("failed to open server preset file: " + path);180    }181 182    std::string contents((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());183 184    static const auto parser = build_peg_parser([](auto & p) {185        // newline ::= "\r\n" / "\n" / "\r"186        auto newline = p.rule("newline", p.literal("\r\n") | p.literal("\n") | p.literal("\r"));187 188        // ws ::= [ \t]*189        auto ws = p.rule("ws", p.chars("[ \t]", 0, -1));190 191        // comment ::= [;#] (!newline .)*192        auto comment = p.rule("comment", p.chars("[;#]", 1, 1) + p.zero_or_more(p.negate(newline) + p.any()));193 194        // eol ::= ws comment? (newline / EOF)195        auto eol = p.rule("eol", ws + p.optional(comment) + (newline | p.end()));196 197        // ident ::= [a-zA-Z_] [a-zA-Z0-9_.-]*198        auto ident = p.rule("ident", p.chars("[a-zA-Z_]", 1, 1) + p.chars("[a-zA-Z0-9_.-]", 0, -1));199 200        // value ::= (!eol-start .)*201        auto eol_start = p.rule("eol-start", ws + (p.chars("[;#]", 1, 1) | newline | p.end()));202        auto value = p.rule("value", p.zero_or_more(p.negate(eol_start) + p.any()));203 204        // header-line ::= "[" ws ident ws "]" eol205        auto header_line = p.rule("header-line", "[" + ws + p.tag("section-name", p.chars("[^]]")) + ws + "]" + eol);206 207        // kv-line ::= ident ws "=" ws value eol208        auto kv_line = p.rule("kv-line", p.tag("key", ident) + ws + "=" + ws + p.tag("value", value) + eol);209 210        // comment-line ::= ws comment (newline / EOF)211        auto comment_line = p.rule("comment-line", ws + comment + (newline | p.end()));212 213        // blank-line ::= ws (newline / EOF)214        auto blank_line = p.rule("blank-line", ws + (newline | p.end()));215 216        // line ::= header-line / kv-line / comment-line / blank-line217        auto line = p.rule("line", header_line | kv_line | comment_line | blank_line);218 219        // ini ::= line* EOF220        auto ini = p.rule("ini", p.zero_or_more(line) + p.end());221 222        return ini;223    });224 225    common_peg_parse_context ctx(contents);226    const auto result = parser.parse(ctx);227    if (!result.success()) {228        throw std::runtime_error("failed to parse server config file: " + path);229    }230 231    std::string current_section = COMMON_PRESET_DEFAULT_NAME;232    std::string current_key;233 234    ctx.ast.visit(result, [&](const auto & node) {235        if (node.tag == "section-name") {236            const std::string section = std::string(node.text);237            current_section = section;238            parsed[current_section] = {};239        } else if (node.tag == "key") {240            const std::string key = std::string(node.text);241            current_key = key;242        } else if (node.tag == "value" && !current_key.empty() && !current_section.empty()) {243            parsed[current_section][current_key] = std::string(node.text);244            current_key.clear();245        }246    });247 248    return parsed;249}250 251static std::map<std::string, common_arg> get_map_key_opt(common_params_context & ctx_params) {252    std::map<std::string, common_arg> mapping;253    for (const auto & opt : ctx_params.options) {254        for (const auto & env : opt.get_env()) {255            mapping[env] = opt;256        }257        for (const auto & arg : opt.get_args()) {258            mapping[rm_leading_dashes(arg)] = opt;259        }260    }261    return mapping;262}263 264static bool is_bool_arg(const common_arg & arg) {265    return !arg.args_neg.empty();266}267 268static std::string parse_bool_arg(const common_arg & arg, const std::string & key, const std::string & value) {269    // if this is a negated arg, we need to reverse the value270    for (const auto & neg_arg : arg.args_neg) {271        if (rm_leading_dashes(neg_arg) == key) {272            return common_arg_utils::is_truthy(value) ? "false" : "true";273        }274    }275    // otherwise, not negated276    return value;277}278 279common_preset_context::common_preset_context(llama_example ex)280        : ctx_params(common_params_parser_init(default_params, ex)) {281    common_params_add_preset_options(ctx_params.options);282    key_to_opt = get_map_key_opt(ctx_params);283}284 285common_presets common_preset_context::load_from_ini(const std::string & path, common_preset & global) const {286    common_presets out;287    auto ini_data = parse_ini_from_file(path);288 289    for (auto section : ini_data) {290        common_preset preset;291        std::string section_name = section.first.empty() ? std::string(COMMON_PRESET_DEFAULT_NAME) : section.first;292        if (section_name != "*" && section_name != COMMON_PRESET_DEFAULT_NAME) {293            auto colon_idx = section_name.rfind(':');294            if (colon_idx != std::string::npos) {295                std::string tag       = section_name.substr(colon_idx + 1);296                std::string canon_tag = canonical_tag(tag);297                if (canon_tag != tag) {298                    section_name = section_name.substr(0, colon_idx + 1) + canon_tag;299                }300            }301        }302        preset.name = section_name;303        LOG_DBG("loading preset: %s\n", preset.name.c_str());304        for (const auto & [key, value] : section.second) {305            if (key == "version") {306                // skip version key (reserved for future use)307                continue;308            }309 310            LOG_DBG("option: %s = %s\n", key.c_str(), value.c_str());311            if (filter_allowed_keys && allowed_keys.find(key) == allowed_keys.end()) {312                throw std::runtime_error(string_format(313                    "option '%s' is not allowed in remote presets",314                    key.c_str()315                ));316            }317            if (key_to_opt.find(key) != key_to_opt.end()) {318                const auto & opt = key_to_opt.at(key);319                if (is_bool_arg(opt)) {320                    preset.options[opt] = parse_bool_arg(opt, key, value);321                } else {322                    preset.options[opt] = value;323                }324                LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());325            } else {326                throw std::runtime_error(string_format(327                    "option '%s' not recognized in preset '%s'",328                    key.c_str(), preset.name.c_str()329                ));330            }331        }332 333        if (preset.name == COMMON_PRESET_DEFAULT_NAME && preset.options.empty()) {334            continue;335        }336 337        if (preset.name == "*") {338            // handle global preset339            global = preset;340        } else {341            out[preset.name] = preset;342        }343    }344 345    return out;346}347 348common_presets common_preset_context::load_from_cache() const {349    common_presets out;350 351    auto cached_models = common_list_cached_models();352    for (const auto & model : cached_models) {353        common_preset preset;354        preset.name = model.to_string();355        preset.set_option(*this, "LLAMA_ARG_HF_REPO", model.to_string());356        out[preset.name] = preset;357    }358 359    return out;360}361 362struct local_model {363    std::string name;364    std::string path;365    std::string path_mmproj;366};367 368common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const {369    if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {370        throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str()));371    }372 373    std::vector<local_model> models;374    auto scan_subdir = [&models](const std::string & subdir_path, const std::string & name) {375        auto files = fs_list(subdir_path, false);376        common_file_info model_file;377        common_file_info first_shard_file;378        common_file_info mmproj_file;379        for (const auto & file : files) {380            if (string_ends_with(file.name, ".gguf")) {381                if (file.name.find("mmproj") != std::string::npos) {382                    mmproj_file = file;383                } else if (file.name.find("-00001-of-") != std::string::npos) {384                    first_shard_file = file;385                } else {386                    model_file = file;387                }388            }389        }390        // single file model391        local_model model{392            /* name        */ name,393            /* path        */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path,394            /* path_mmproj */ mmproj_file.path // can be empty395        };396        if (!model.path.empty()) {397            models.push_back(model);398        }399    };400 401    auto files = fs_list(models_dir, true);402    for (const auto & file : files) {403        if (file.is_dir) {404            scan_subdir(file.path, file.name);405        } else if (string_ends_with(file.name, ".gguf")) {406            // single file model407            std::string name = file.name;408            string_replace_all(name, ".gguf", "");409            local_model model{410                /* name        */ name,411                /* path        */ file.path,412                /* path_mmproj */ ""413            };414            models.push_back(model);415        }416    }417 418    // convert local models to presets419    common_presets out;420    for (const auto & model : models) {421        common_preset preset;422        preset.name = model.name;423        preset.set_option(*this, "LLAMA_ARG_MODEL", model.path);424        if (!model.path_mmproj.empty()) {425            preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj);426        }427        out[preset.name] = preset;428    }429 430    return out;431}432 433common_preset common_preset_context::load_from_args(int argc, char ** argv) const {434    common_preset preset;435    preset.name = COMMON_PRESET_DEFAULT_NAME;436 437    bool ok = common_params_to_map(argc, argv, ctx_params.ex, preset.options);438    if (!ok) {439        throw std::runtime_error("failed to parse CLI arguments into preset");440    }441 442    return preset;443}444 445common_presets common_preset_context::cascade(const common_presets & base, const common_presets & added) const {446    common_presets out = base; // copy447    for (const auto & [name, preset_added] : added) {448        if (out.find(name) != out.end()) {449            // if exists, merge450            common_preset & target = out[name];451            target.merge(preset_added);452        } else {453            // otherwise, add directly454            out[name] = preset_added;455        }456    }457    return out;458}459 460common_presets common_preset_context::cascade(const common_preset & base, const common_presets & presets) const {461    common_presets out;462    for (const auto & [name, preset] : presets) {463        common_preset tmp = base; // copy464        tmp.name = name;465        tmp.merge(preset);466        out[name] = std::move(tmp);467    }468    return out;469}470