Brunobkr/llama.cpp_AlgMor24_github

1#include "arg.h"2#include "common.h"3#include "download.h"4#include "llama.h"5#include "speculative.h"6 7#include <limits>8#include <string>9#include <vector>10#include <sstream>11#include <unordered_set>12 13#undef NDEBUG14#include <cassert>15 16static void test(void) {17    common_params params;18 19    auto assert_output_limits = [](int32_t n_batch, int32_t n_parallel, int32_t n_draft,20                                   int32_t total, int32_t per_seq) {21        const auto limits = common_speculative_get_output_limits(n_batch, n_parallel, n_draft);22        assert(limits.total == total);23        assert(limits.per_seq == per_seq);24    };25 26    assert_output_limits(16, 2,  3, 8, 4);27    assert_output_limits(16, 2, -1, 2, 1);28    assert_output_limits( 6, 2,  3, 6, 4);29    assert_output_limits( 2, 1,  3, 2, 2);30    assert_output_limits(31            std::numeric_limits<int32_t>::max(),32            std::numeric_limits<int32_t>::max(),33            std::numeric_limits<int32_t>::max(),34            std::numeric_limits<int32_t>::max(),35            std::numeric_limits<int32_t>::max());36 37    {38        common_params base;39        base.n_parallel = 4;40        base.n_outputs_max_per_seq = 8;41 42        const auto draft = common_base_params_to_speculative(base);43        assert(draft.n_outputs_max == 4);44        assert(draft.n_outputs_max_per_seq == 1);45    }46 47    printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");48    for (int ex = 0; ex < LLAMA_EXAMPLE_COUNT; ex++) {49        try {50            auto ctx_arg = common_params_parser_init(params, (enum llama_example)ex);51            common_params_add_preset_options(ctx_arg.options);52            std::unordered_set<std::string> seen_args;53            std::unordered_set<std::string> seen_env_vars;54            for (const auto & opt : ctx_arg.options) {55                // check for args duplications56                for (const auto & arg : opt.get_args()) {57                    if (seen_args.find(arg) == seen_args.end()) {58                        seen_args.insert(arg);59                    } else {60                        fprintf(stderr, "test-arg-parser: found different handlers for the same argument: %s", arg.c_str());61                        exit(1);62                    }63                }64                // check for env var duplications65                for (const auto & env : opt.get_env()) {66                    if (seen_env_vars.find(env) == seen_env_vars.end()) {67                        seen_env_vars.insert(env);68                    } else {69                        fprintf(stderr, "test-arg-parser: found different handlers for the same env var: %s", env.c_str());70                        exit(1);71                    }72                }73 74                // exclude spec args from this check75                // ref: https://github.com/ggml-org/llama.cpp/pull/2239776                const bool skip = opt.is_spec;77 78                // ensure shorter argument precedes longer argument79                if (!skip && opt.args.size() > 1) {80                    const std::string first(opt.args.front());81                    const std::string last(opt.args.back());82 83                    if (first.length() > last.length()) {84                        fprintf(stderr, "test-arg-parser: shorter argument should come before longer one: %s, %s\n",85                                first.c_str(), last.c_str());86                        assert(false);87                    }88                }89 90                // same check for negated arguments91                if (opt.args_neg.size() > 1) {92                    const std::string first(opt.args_neg.front());93                    const std::string last(opt.args_neg.back());94 95                    if (first.length() > last.length()) {96                        fprintf(stderr, "test-arg-parser: shorter negated argument should come before longer one: %s, %s\n",97                                first.c_str(), last.c_str());98                        assert(false);99                    }100                }101            }102        } catch (std::exception & e) {103            printf("%s\n", e.what());104            assert(false);105        }106    }107 108    auto list_str_to_char = [](std::vector<std::string> & argv) -> std::vector<char *> {109        std::vector<char *> res;110        for (auto & arg : argv) {111            res.push_back(const_cast<char *>(arg.data()));112        }113        return res;114    };115 116    std::vector<std::string> argv;117 118    printf("test-arg-parser: test invalid usage\n\n");119 120    // missing value121    argv = {"binary_name", "-m"};122    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));123 124    // wrong value (int)125    argv = {"binary_name", "-ngl", "hello"};126    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));127 128    // wrong value (enum)129    argv = {"binary_name", "-sm", "hello"};130    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));131 132    {133        common_params penalty_params;134        assert(penalty_params.sampling.penalty_last_n == 64);135        assert(penalty_params.sampling.dry_penalty_last_n == 64);136 137        argv = {"binary_name", "--repeat-last-n", "-1"};138        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));139 140        argv = {"binary_name", "--dry-penalty-last-n", "-1"};141        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));142 143        argv = {"binary_name", "--repeat-penalty", "0"};144        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));145 146        argv = {"binary_name", "--repeat-penalty", "-1"};147        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));148 149        argv = {"binary_name", "--repeat-penalty", "nan"};150        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));151 152        argv = {"binary_name", "--repeat-penalty", "inf"};153        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));154 155        argv = {"binary_name", "--repeat-penalty", "-inf"};156        assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));157 158        const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"};159        const char * nonfinite_values[] = {"nan", "inf", "-inf"};160        for (const char * option : penalty_options) {161            for (const char * value : nonfinite_values) {162                argv = {"binary_name", option, value};163                assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));164            }165        }166    }167 168    // non-existence arg in specific example (--draft cannot be used outside llama-speculative)169    argv = {"binary_name", "--draft", "123"};170    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));171 172    argv = {"binary_name", "-lm", "hello"};173    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));174 175    printf("test-arg-parser: test valid usage\n\n");176 177    argv = {"binary_name", "-m", "model_file.gguf"};178    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));179    assert(params.model.path == "model_file.gguf");180 181    argv = {"binary_name", "-t", "1234"};182    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));183    assert(params.cpuparams.n_threads == 1234);184 185    argv = {"binary_name", "--verbose"};186    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));187    assert(params.verbosity > 1);188 189    argv = {"binary_name", "-m", "abc.gguf", "--predict", "6789", "--batch-size", "9090"};190    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));191    assert(params.model.path == "abc.gguf");192    assert(params.n_predict == 6789);193    assert(params.n_batch == 9090);194 195    // --draft cannot be used outside llama-speculative196    argv = {"binary_name", "--spec-draft-n-max", "123"};197    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));198    assert(params.speculative.draft.n_max == 123);199 200    argv = {"binary_name", "-lm", "none"};201    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));202    assert(params.load_mode == LLAMA_LOAD_MODE_NONE);203 204    argv = {"binary_name", "-lm", "mmap"};205    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));206    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);207 208    argv = {"binary_name", "-lm", "mlock"};209    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));210    assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);211 212    argv = {"binary_name", "-lm", "mmap+mlock"};213    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));214    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);215 216    argv = {"binary_name", "-lm", "dio"};217    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));218    assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);219 220    // multi-value args (CSV)221    argv = {"binary_name", "--lora", "file1.gguf,\"file2,2.gguf\",\"file3\"\"3\"\".gguf\",file4\".gguf"};222    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));223    assert(params.lora_adapters.size() == 4);224    assert(params.lora_adapters[0].path == "file1.gguf");225    assert(params.lora_adapters[1].path == "file2,2.gguf");226    assert(params.lora_adapters[2].path == "file3\"3\".gguf");227    assert(params.lora_adapters[3].path == "file4\".gguf");228 229// skip this part on windows, because setenv is not supported230#ifdef _WIN32231    printf("test-arg-parser: skip on windows build\n");232#else233    printf("test-arg-parser: test environment variables (valid + invalid usages)\n\n");234 235    setenv("LLAMA_ARG_THREADS", "blah", true);236    argv = {"binary_name"};237    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));238 239    setenv("LLAMA_ARG_MODEL", "blah.gguf", true);240    setenv("LLAMA_ARG_THREADS", "1010", true);241    argv = {"binary_name"};242    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));243    assert(params.model.path == "blah.gguf");244    assert(params.cpuparams.n_threads == 1010);245 246    setenv("LLAMA_ARG_LOAD_MODE", "blah", true);247    argv = {"binary_name"};248    assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));249 250    setenv("LLAMA_ARG_LOAD_MODE", "mmap", true);251    argv = {"binary_name"};252    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));253    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);254 255    setenv("LLAMA_ARG_LOAD_MODE", "mlock", true);256    argv = {"binary_name"};257    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));258    assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);259 260    setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true);261    argv = {"binary_name"};262    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));263    assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);264 265    setenv("LLAMA_ARG_LOAD_MODE", "dio", true);266    argv = {"binary_name"};267    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));268    assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);269 270    printf("test-arg-parser: test negated environment variables\n\n");271 272    setenv("LLAMA_ARG_LOAD_MODE", "none", true);273    setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format274    argv = {"binary_name"};275    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));276    assert(params.load_mode == LLAMA_LOAD_MODE_NONE);277    assert(params.no_perf == true);278 279    printf("test-arg-parser: test environment variables being overwritten\n\n");280 281    setenv("LLAMA_ARG_MODEL", "blah.gguf", true);282    setenv("LLAMA_ARG_THREADS", "1010", true);283    argv = {"binary_name", "-m", "overwritten.gguf"};284    assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));285    assert(params.model.path == "overwritten.gguf");286    assert(params.cpuparams.n_threads == 1010);287#endif // _WIN32288 289    printf("test-arg-parser: test download functions\n\n");290    const char * GOOD_URL = "http://ggml.ai/";291    const char * BAD_URL  = "http://ggml.ai/404";292 293    {294        printf("test-arg-parser: test good URL\n\n");295        auto res = common_remote_get_content(GOOD_URL, {});296        assert(res.first == 200);297        assert(res.second.size() > 0);298        std::string str(res.second.data(), res.second.size());299        assert(str.find("llama.cpp") != std::string::npos);300    }301 302    {303        printf("test-arg-parser: test bad URL\n\n");304        auto res = common_remote_get_content(BAD_URL, {});305        assert(res.first == 404);306    }307 308    {309        printf("test-arg-parser: test max size error\n");310        common_remote_params params;311        params.max_size = 1;312        try {313            common_remote_get_content(GOOD_URL, params);314            assert(false && "it should throw an error");315        } catch (std::exception & e) {316            printf("  expected error: %s\n\n", e.what());317        }318    }319 320    printf("test-arg-parser: all tests OK\n\n");321}322 323int main(void) {324    try {325        test();326    } catch (std::exception & e) {327        fprintf(stderr, "test-arg-parser: exception: %s\n", e.what());328        return 1;329    }330    return 0;331}332