Brunobkr/llama.cpp_AlgMor24_github

1#include "llama.h"2 3#include "llama-impl.h"4 5#include "llama-chat.h"6#include "llama-context.h"7#include "llama-mmap.h"8#include "llama-vocab.h"9#include "llama-model-loader.h"10#include "llama-model-saver.h"11#include "llama-model.h"12 13#include "ggml.h"14#include "ggml-cpp.h"15#include "ggml-backend.h"16#include "gguf.h"17 18#include <algorithm>19#include <cassert>20#include <cinttypes>21#include <cstddef>22#include <cstdint>23#include <cstdio>24#include <cstring>25#include <ctime>26#include <stdexcept>27#include <vector>28 29#if defined(_MSC_VER)30#pragma warning(disable: 4244 4267) // possible loss of data31#endif32 33//34// interface implementation35//36 37const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type) {38    switch (flash_attn_type) {39        case LLAMA_FLASH_ATTN_TYPE_AUTO:40            return "auto";41        case LLAMA_FLASH_ATTN_TYPE_DISABLED:42            return "disabled";43        case LLAMA_FLASH_ATTN_TYPE_ENABLED:44            return "enabled";45    }46    GGML_ABORT("fatal error");47}48 49const char * llama_load_mode_name(enum llama_load_mode load_mode) {50    switch (load_mode) {51        case LLAMA_LOAD_MODE_NONE:52            return "none";53        case LLAMA_LOAD_MODE_MMAP:54            return "mmap";55        case LLAMA_LOAD_MODE_MLOCK:56            return "mlock";57        case LLAMA_LOAD_MODE_MMAP_MLOCK:58            return "mmap+mlock";59        case LLAMA_LOAD_MODE_DIRECT_IO:60            return "dio";61    }62    GGML_ABORT("fatal error");63}64 65enum llama_load_mode llama_load_mode_from_str(const char * str) {66    if (std::strcmp(str, "none") == 0)       { return LLAMA_LOAD_MODE_NONE;       }67    if (std::strcmp(str, "mmap") == 0)       { return LLAMA_LOAD_MODE_MMAP;       }68    if (std::strcmp(str, "mlock") == 0)      { return LLAMA_LOAD_MODE_MLOCK;      }69    if (std::strcmp(str, "mmap+mlock") == 0) { return LLAMA_LOAD_MODE_MMAP_MLOCK; }70    if (std::strcmp(str, "dio") == 0)        { return LLAMA_LOAD_MODE_DIRECT_IO;  }71    throw std::invalid_argument(std::string("unknown load mode: ") + str);72}73 74struct llama_sampler_chain_params llama_sampler_chain_default_params() {75    struct llama_sampler_chain_params result = {76        /*.no_perf =*/ true,77    };78 79    return result;80}81 82size_t llama_max_devices(void) {83    return 16;84}85 86size_t llama_max_tensor_buft_overrides() {87    return 4096;88}89 90bool llama_supports_mmap(void) {91    return llama_mmap::SUPPORTED;92}93 94bool llama_supports_mlock(void) {95    return llama_mlock::SUPPORTED;96}97 98bool llama_supports_gpu_offload(void) {99    if (!ggml_backend_reg_count()) {100        ggml_backend_load_all();101    }102    return ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) != nullptr ||103           ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU) != nullptr ||104           llama_supports_rpc();105}106 107bool llama_supports_rpc(void) {108    if (!ggml_backend_reg_count()) {109        ggml_backend_load_all();110    }111    return ggml_backend_reg_by_name("RPC") != nullptr;112}113 114void llama_backend_init(void) {115    ggml_time_init();116 117    // needed to initialize f16 tables118    {119        struct ggml_init_params params = { 0, NULL, false };120        struct ggml_context * ctx = ggml_init(params);121        ggml_free(ctx);122    }123 124    if (!ggml_backend_reg_count()) {125        ggml_backend_load_all();126    }127}128 129void llama_numa_init(enum ggml_numa_strategy numa) {130    if (numa != GGML_NUMA_STRATEGY_DISABLED) {131        auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);132        GGML_ASSERT(dev && "CPU backend is not loaded");133        auto * reg = ggml_backend_dev_backend_reg(dev);134        auto * numa_init_fn = (decltype(ggml_numa_init) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_numa_init");135        if (numa_init_fn) {136            numa_init_fn(numa);137        }138    }139}140 141void llama_backend_free(void) {142    ggml_quantize_free();143}144 145int64_t llama_time_us(void) {146    return ggml_time_us();147}148 149// returns true on success150static bool llama_prepare_model_devices(const llama_model_params & params, llama_model * model) {151    // create list of devices to use with this model152    if (params.devices) {153        if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {154            size_t n_devs = 0;155            while (params.devices[n_devs]) {156                n_devs++;157            }158            if (n_devs == 0) {159                LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);160                return false;161            }162            LLAMA_LOG_INFO("%s: creating a Meta device with %zu devices\n", __func__, n_devs);163            for (size_t i = 0; i < n_devs; ++i) {164                LLAMA_LOG_INFO("%s: - device %zu: %s\n", __func__, i, ggml_backend_dev_name(params.devices[i]));165            }166            model->get_split_state_ud.n_devices = n_devs;167            model->get_split_state_ud.model = model;168            model->devices.push_back({169                true, ggml_backend_meta_device(170                params.devices, n_devs, llama_meta_device_get_split_state, &model->get_split_state_ud)171            });172        } else {173            for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {174                model->devices.push_back({false, *dev});175            }176        }177    } else {178        // default device selection179 180        // build list of available devices181        std::vector<llama_device> gpus;182        std::vector<llama_device> igpus;183        std::vector<llama_device> rpc_servers;184 185        if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {186            std::vector<ggml_backend_dev_t> devs;187            devs.reserve(ggml_backend_dev_count());188            for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {189                auto * dev = ggml_backend_dev_get(i);190                if (ggml_backend_dev_buffer_type(dev) == ggml_backend_cpu_buffer_type()) {191                    LLAMA_LOG_INFO("%s: skipping %s (%s) for tensor parallelism\n", __func__, ggml_backend_dev_name(dev), ggml_backend_dev_description(dev));192                    continue;193                }194                devs.push_back(dev);195            }196            if (devs.empty()) {197                LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);198                return false;199            }200 201            LLAMA_LOG_INFO("%s: creating a Meta device for tensor parallelism from %zu devices:\n", __func__, devs.size());202            for (size_t i = 0; i < devs.size(); ++i) {203                LLAMA_LOG_INFO("%s: - device %zu: %s (%s)\n", __func__, i, ggml_backend_dev_name(devs[i]), ggml_backend_dev_description(devs[i]));204            }205 206            GGML_ASSERT(!devs.empty());207            model->get_split_state_ud.n_devices = devs.size();208            model->get_split_state_ud.model     = model;209            gpus.push_back({210                true, ggml_backend_meta_device(211                devs.data(), devs.size(), llama_meta_device_get_split_state, &model->get_split_state_ud)212            });213        } else {214            for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {215                ggml_backend_dev_t dev = ggml_backend_dev_get(i);216                switch (ggml_backend_dev_type(dev)) {217                    case GGML_BACKEND_DEVICE_TYPE_CPU:218                    case GGML_BACKEND_DEVICE_TYPE_ACCEL:219                        // skip CPU backends since they are handled separately220                        break;221 222                    case GGML_BACKEND_DEVICE_TYPE_GPU: {223                        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);224                        if (ggml_backend_reg_name(reg) == std::string("RPC")) {225                            rpc_servers.push_back({false, dev});226                        } else {227                            // check if there is already a GPU with the same device id228                            ggml_backend_dev_props props;229                            ggml_backend_dev_get_props(dev, &props);230                            auto it = std::find_if(gpus.begin(), gpus.end(), [&props](const llama_device & d) {231                                ggml_backend_dev_props d_props;232                                ggml_backend_dev_get_props(d.dev, &d_props);233                                if (props.device_id && d_props.device_id) {234                                    return strcmp(props.device_id, d_props.device_id) == 0;235                                }236                                return false;237                            });238 239                            if (it != gpus.end()) {240                                LLAMA_LOG_INFO("%s: skipping device %s (%s) with id %s - already using device %s (%s) with the same id\n",241                                        __func__,242                                        ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),243                                        props.device_id ? props.device_id : "unknown id",244                                        ggml_backend_dev_name(it->dev), ggml_backend_dev_description(it->dev));245                            } else {246                                gpus.push_back({false, dev});247                            }248                        }249                        break;250                    }251 252                    case GGML_BACKEND_DEVICE_TYPE_IGPU:253                        if (igpus.empty()) {254                            igpus.push_back({false, dev});255                        }256                        break;257                    case GGML_BACKEND_DEVICE_TYPE_META:258                        GGML_ABORT("fatal error");259                }260            }261        }262 263        // add RPC servers at the front of the list to minimize network transfers264        model->devices.insert(model->devices.begin(), rpc_servers.begin(), rpc_servers.end());265 266        // add GPUs267        model->devices.insert(model->devices.end(), gpus.begin(), gpus.end());268 269        // add integrated GPUs only if no discrete GPUs were found270        // (RPC servers do not count, otherwise the local iGPU would be dropped on iGPU+RPC setups)271        if (gpus.empty()) {272            model->devices.insert(model->devices.end(), igpus.begin(), igpus.end());273        }274    }275 276    // if using single GPU mode, remove all except the main GPU277    if (params.split_mode == LLAMA_SPLIT_MODE_NONE && !model->devices.empty()) {278        if (params.main_gpu < 0) {279            model->devices.clear();280        } else {281            if (params.main_gpu >= (int)model->devices.size()) {282                LLAMA_LOG_ERROR("%s: invalid value for main_gpu: %d (available devices: %zu)\n", __func__, params.main_gpu, model->devices.size());283                return false;284            }285            llama_device main_gpu = model->devices[params.main_gpu];286            model->devices.clear();287            model->devices.push_back(main_gpu);288        }289    }290 291    for (const auto & dev : model->devices) {292        ggml_backend_dev_props props;293        ggml_backend_dev_get_props(dev.dev, &props);294        LLAMA_LOG_INFO("%s: using device %s (%s) (%s) - %zu MiB free\n", __func__,295                ggml_backend_dev_name(dev.dev), ggml_backend_dev_description(dev.dev),296                props.device_id ? props.device_id : "unknown id",297                props.memory_free/1024/1024);298    }299 300    return true;301}302 303// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback304static std::pair<int, llama_model *> llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,305        const std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model_params & params) {306    try {307        llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,308            params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);309 310        ml.print_info();311        std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));312 313        bool ok = llama_prepare_model_devices(params, model_ptr.get());314        if (!ok) {315            return {-1, nullptr};316        }317 318        auto * model = dynamic_cast<llama_model_base *>(model_ptr.get());319        if (model == nullptr) {320            GGML_ABORT("fatal error: model does not implement llama_model_base");321        }322 323        // loading time will be recalculated after the first eval, so324        // we take page faults deferred by mmap() into consideration325        model->t_load_us = 0;326        time_meas tm(model->t_load_us);327 328        model->t_start_us = tm.t_start_us;329 330        model->hparams.vocab_only = params.vocab_only;331        model->hparams.no_alloc   = params.no_alloc;332 333        try {334            model->load_hparams(ml);335        } catch(const std::exception & e) {336            throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what()));337        }338        if (model->arch == LLM_ARCH_CLIP) {339            throw std::runtime_error("CLIP cannot be used as main model, use it with --mmproj instead");340        }341        try {342            model->load_vocab(ml);343        } catch(const std::exception & e) {344            throw std::runtime_error("error loading model vocabulary: " + std::string(e.what()));345        }346 347        model->load_stats(ml);348        model->print_info();349 350        if (params.vocab_only) {351            LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__);352            return {0, model_ptr.release()};353        }354 355        if (!model->load_tensors(ml)) {356            return {-2, nullptr};357        }358 359        return {0, model_ptr.release()};360    } catch (const std::exception & err) {361        LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());362        return {-1, nullptr};363    }364}365 366static struct llama_model * llama_model_load_from_file_impl(367        struct gguf_context * metadata,368        llama_model_set_tensor_data_t set_tensor_data,369        void * set_tensor_data_ud,370        const std::string & path_model,371        std::vector<std::string> & splits,372        FILE * file,373        struct llama_model_params params) {374    {375        int n_sources_defined = 0;376        if (metadata != nullptr) {377            n_sources_defined++;378        }379        if (!path_model.empty()) {380            n_sources_defined++;381        }382        if (file != nullptr) {383            n_sources_defined++;384        }385        if (n_sources_defined != 1) {386            LLAMA_LOG_ERROR("%s: exactly one out metadata, path_model, and file must be defined\n", __func__);387            return nullptr;388        }389    }390    ggml_time_init();391 392    if (!params.vocab_only && ggml_backend_reg_count() == 0) {393        LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__);394        return nullptr;395    }396 397    unsigned cur_percentage = 0;398    if (params.progress_callback == NULL) {399        params.progress_callback_user_data = &cur_percentage;400        params.progress_callback = [](float progress, void * ctx) {401            unsigned * cur_percentage_p = (unsigned *) ctx;402            unsigned percentage = (unsigned) (100 * progress);403            while (percentage > *cur_percentage_p) {404                *cur_percentage_p = percentage;405                LLAMA_LOG_CONT(".");406                if (percentage >= 100) {407                    LLAMA_LOG_CONT("\n");408                }409            }410            return true;411        };412    }413 414    const auto [status, model] = llama_model_load(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, file, params);415    GGML_ASSERT(status <= 0);416    if (status < 0) {417        if (status == -1) {418            LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);419        } else if (status == -2) {420            LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);421        }422 423        if (model) {424            llama_model_free(model);425        }426        return nullptr;427    }428 429    return model;430}431 432struct llama_model * llama_model_init_from_user(433        struct gguf_context * metadata,434        llama_model_set_tensor_data_t set_tensor_data,435        void * set_tensor_data_ud,436        struct llama_model_params params) {437    GGML_ASSERT(metadata != nullptr);438    std::string path_model;439    std::vector<std::string> splits = {};440    params.load_mode = LLAMA_LOAD_MODE_NONE;441    params.use_extra_bufts = false;442    return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params);443}444// deprecated445struct llama_model * llama_load_model_from_file(446        const char * path_model,447        struct llama_model_params params) {448    return llama_model_load_from_file(path_model, params);449}450 451struct llama_model * llama_model_load_from_file(452        const char * path_model,453        struct llama_model_params params) {454    std::vector<std::string> splits = {};455    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, /*file*/ nullptr, params);456}457 458struct llama_model * llama_model_load_from_splits(459        const char ** paths,460        size_t n_paths,461        struct llama_model_params params) {462    std::vector<std::string> splits;463    if (n_paths == 0) {464        LLAMA_LOG_ERROR("%s: list of splits is empty\n", __func__);465        return nullptr;466    }467    splits.reserve(n_paths);468    for (size_t i = 0; i < n_paths; ++i) {469        splits.push_back(paths[i]);470    }471    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, splits.front(), splits, /*file*/ nullptr, params);472}473 474struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_model_params params) {475    if (!file) {476        LLAMA_LOG_ERROR("%s: file is NULL\n", __func__);477        return nullptr;478    }479    std::string path_model;480    std::vector<std::string> splits = {};481    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, file, params);482}483 484void llama_model_save_to_file(const struct llama_model * model, const char * path_model) {485    llama_model_saver ms(model);486    ms.add_kv_from_model();487    ms.add_tensors_from_model();488    ms.save(path_model);489}490 491//492// chat templates493//494 495int32_t llama_chat_apply_template(496                              const char * tmpl,497         const struct llama_chat_message * chat,498                                  size_t   n_msg,499                                    bool   add_ass,500                                    char * buf,501                                 int32_t   length) {502    const std::string curr_tmpl(tmpl == nullptr ? "chatml" : tmpl);503 504    // format the chat to string505    std::vector<const llama_chat_message *> chat_vec;506    chat_vec.resize(n_msg);507    for (size_t i = 0; i < n_msg; i++) {508        chat_vec[i] = &chat[i];509    }510 511    std::string formatted_chat;512    llm_chat_template detected_tmpl = llm_chat_detect_template(curr_tmpl);513    if (detected_tmpl == LLM_CHAT_TEMPLATE_UNKNOWN) {514        return -1;515    }516    int32_t res = llm_chat_apply_template(detected_tmpl, chat_vec, formatted_chat, add_ass);517    if (res < 0) {518        return res;519    }520    if (buf && length > 0) {521        strncpy(buf, formatted_chat.c_str(), length);522    }523    return res;524}525 526//527// model split528//529 530int32_t llama_split_path(531    char * split_path,532    size_t maxlen,533    const char * path_prefix,534    int32_t split_no,535    int32_t split_count) {536 537    static const char * const SPLIT_PATH_FORMAT = "%s-%05d-of-%05d.gguf";538 539    const int written = snprintf(540        split_path,541        maxlen,542        SPLIT_PATH_FORMAT,543        path_prefix,544        split_no + 1,545        split_count546    );547 548    if (written < 0 || (size_t) written >= maxlen) {549        return 0;550    }551 552    return (int32_t) written;553}554 555int32_t llama_split_prefix(556    char * split_prefix,557    size_t maxlen,558    const char * split_path,559    int32_t split_no,560    int32_t split_count) {561 562    const std::string str_split_path(split_path);563 564    char postfix[32];565    snprintf(postfix, sizeof(postfix), "-%05d-of-%05d.gguf", split_no + 1, split_count);566 567    const std::string str_postfix(postfix);568    if (str_split_path.size() <= str_postfix.size()) {569        return 0;570    }571 572    const size_t size_prefix = str_split_path.size() - str_postfix.size();573 574    if (str_split_path.compare(size_prefix, std::string::npos, str_postfix) == 0) {575        const size_t copy_len = std::min(size_prefix + 1, maxlen);576        snprintf(split_prefix, copy_len, "%s", split_path);577 578        return (int32_t) size_prefix;579    }580 581    return 0;582}583 584const char * llama_print_system_info(void) {585    static std::string s;586    s.clear(); // Clear the string, since it's static, otherwise it will accumulate data from previous calls.587 588    for (size_t i = 0; i < ggml_backend_reg_count(); i++) {589        auto * reg = ggml_backend_reg_get(i);590        auto * get_features_fn = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");591        if (get_features_fn) {592            ggml_backend_feature * features = get_features_fn(reg);593            s += ggml_backend_reg_name(reg);594            s += " : ";595            for (; features->name; features++) {596                s += features->name;597                s += " = ";598                s += features->value;599                s += " | ";600            }601        }602    }603 604    return s.c_str();605}606 607