echodict/llama.cpp

1#include "llama.h"2 3#include "llama-impl.h"4 5#include "llama-chat.h"6#include "llama-context.h"7#include "llama-mmap.h"8#include "llama-vocab.h"9#include "llama-model-loader.h"10#include "llama-model-saver.h"11#include "llama-model.h"12 13#include "ggml.h"14#include "ggml-cpp.h"15#include "ggml-backend.h"16#include "gguf.h"17 18#include <algorithm>19#include <cassert>20#include <cinttypes>21#include <cstddef>22#include <cstdint>23#include <cstdio>24#include <cstring>25#include <ctime>26#include <stdexcept>27#include <vector>28 29#if defined(_MSC_VER)30#pragma warning(disable: 4244 4267) // possible loss of data31#endif32 33//34// interface implementation35//36 37const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type) {38    switch (flash_attn_type) {39        case LLAMA_FLASH_ATTN_TYPE_AUTO:40            return "auto";41        case LLAMA_FLASH_ATTN_TYPE_DISABLED:42            return "disabled";43        case LLAMA_FLASH_ATTN_TYPE_ENABLED:44            return "enabled";45    }46    GGML_ABORT("fatal error");47}48 49struct llama_sampler_chain_params llama_sampler_chain_default_params() {50    struct llama_sampler_chain_params result = {51        /*.no_perf =*/ true,52    };53 54    return result;55}56 57size_t llama_max_devices(void) {58    return 16;59}60 61size_t llama_max_tensor_buft_overrides() {62    return 4096;63}64 65bool llama_supports_mmap(void) {66    return llama_mmap::SUPPORTED;67}68 69bool llama_supports_mlock(void) {70    return llama_mlock::SUPPORTED;71}72 73bool llama_supports_gpu_offload(void) {74    return ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) != nullptr ||75           ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU) != nullptr ||76           llama_supports_rpc();77}78 79bool llama_supports_rpc(void) {80    return ggml_backend_reg_by_name("RPC") != nullptr;81}82 83void llama_backend_init(void) {84    ggml_time_init();85 86    // needed to initialize f16 tables87    {88        struct ggml_init_params params = { 0, NULL, false };89        struct ggml_context * ctx = ggml_init(params);90        ggml_free(ctx);91    }92}93 94void llama_numa_init(enum ggml_numa_strategy numa) {95    if (numa != GGML_NUMA_STRATEGY_DISABLED) {96        auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);97        GGML_ASSERT(dev && "CPU backend is not loaded");98        auto * reg = ggml_backend_dev_backend_reg(dev);99        auto * numa_init_fn = (decltype(ggml_numa_init) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_numa_init");100        if (numa_init_fn) {101            numa_init_fn(numa);102        }103    }104}105 106void llama_backend_free(void) {107    ggml_quantize_free();108}109 110int64_t llama_time_us(void) {111    return ggml_time_us();112}113 114// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback115static int llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,116        const std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model & model, llama_model_params & params) {117    // loading time will be recalculated after the first eval, so118    // we take page faults deferred by mmap() into consideration119    model.t_load_us = 0;120    time_meas tm(model.t_load_us);121 122    model.t_start_us = tm.t_start_us;123 124    try {125        llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.use_mmap, params.use_direct_io,126            params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides);127 128        ml.print_info();129 130        model.hparams.vocab_only = params.vocab_only;131        model.hparams.no_alloc   = params.no_alloc;132 133        try {134            model.load_arch(ml);135        } catch(const std::exception & e) {136            throw std::runtime_error("error loading model architecture: " + std::string(e.what()));137        }138        try {139            model.load_hparams(ml);140        } catch(const std::exception & e) {141            throw std::runtime_error("error loading model hyperparameters: " + std::string(e.what()));142        }143        if (model.arch == LLM_ARCH_CLIP) {144            throw std::runtime_error("CLIP cannot be used as main model, use it with --mmproj instead");145        }146        try {147            model.load_vocab(ml);148        } catch(const std::exception & e) {149            throw std::runtime_error("error loading model vocabulary: " + std::string(e.what()));150        }151 152        model.load_stats(ml);153        model.print_info();154 155        if (params.vocab_only) {156            LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__);157            return 0;158        }159 160        if (!model.load_tensors(ml)) {161            return -2;162        }163    } catch (const std::exception & err) {164        LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());165        return -1;166    }167 168    return 0;169}170 171static struct llama_model * llama_model_load_from_file_impl(172        struct gguf_context * metadata,173        llama_model_set_tensor_data_t set_tensor_data,174        void * set_tensor_data_ud,175        const std::string & path_model,176        std::vector<std::string> & splits,177        FILE * file,178        struct llama_model_params params) {179    {180        int n_sources_defined = 0;181        if (metadata != nullptr) {182            n_sources_defined++;183        }184        if (!path_model.empty()) {185            n_sources_defined++;186        }187        if (file != nullptr) {188            n_sources_defined++;189        }190        if (n_sources_defined != 1) {191            LLAMA_LOG_ERROR("%s: exactly one out metadata, path_model, and file must be defined\n", __func__);192            return nullptr;193        }194    }195    ggml_time_init();196 197    if (!params.vocab_only && ggml_backend_reg_count() == 0) {198        LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__);199        return nullptr;200    }201 202    unsigned cur_percentage = 0;203    if (params.progress_callback == NULL) {204        params.progress_callback_user_data = &cur_percentage;205        params.progress_callback = [](float progress, void * ctx) {206            unsigned * cur_percentage_p = (unsigned *) ctx;207            unsigned percentage = (unsigned) (100 * progress);208            while (percentage > *cur_percentage_p) {209                *cur_percentage_p = percentage;210                LLAMA_LOG_CONT(".");211                if (percentage >= 100) {212                    LLAMA_LOG_CONT("\n");213                }214            }215            return true;216        };217    }218 219    llama_model * model = new llama_model(params);220 221    // create list of devices to use with this model222    if (params.devices) {223        if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {224            size_t n_devs = 0;225            while (params.devices[n_devs]) {226                n_devs++;227            }228            if (n_devs == 0) {229                LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);230                return nullptr;231            }232            LLAMA_LOG_INFO("%s: creating a Meta device with %zu devices\n", __func__, n_devs);233            for (size_t i = 0; i < n_devs; ++i) {234                LLAMA_LOG_INFO("%s: - device %zu: %s\n", __func__, i, ggml_backend_dev_name(params.devices[i]));235            }236            model->get_split_state_ud.n_devices = n_devs;237            model->get_split_state_ud.model = model;238            model->devices.push_back({239                true, ggml_backend_meta_device(240                params.devices, n_devs, llama_meta_device_get_split_state, &model->get_split_state_ud)241            });242        } else {243            for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {244                model->devices.push_back({false, *dev});245            }246        }247    } else {248        // default device selection249 250        // build list of available devices251        std::vector<llama_device> gpus;252        std::vector<llama_device> igpus;253        std::vector<llama_device> rpc_servers;254 255        if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR) {256            std::vector<ggml_backend_dev_t> devs;257            devs.reserve(ggml_backend_dev_count());258            for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {259                auto * dev = ggml_backend_dev_get(i);260                if (ggml_backend_dev_buffer_type(dev) == ggml_backend_cpu_buffer_type()) {261                    LLAMA_LOG_INFO("%s: skipping %s (%s) for tensor parallelism\n", __func__, ggml_backend_dev_name(dev), ggml_backend_dev_description(dev));262                    continue;263                }264                devs.push_back(dev);265            }266            if (devs.empty()) {267                LLAMA_LOG_ERROR("%s: LLAMA_SPLIT_MODE_TENSOR needs >= 1 devices\n", __func__);268                return nullptr;269            }270 271            LLAMA_LOG_INFO("%s: creating a Meta device for tensor parallelism from %zu devices:\n", __func__, devs.size());272            for (size_t i = 0; i < devs.size(); ++i) {273                LLAMA_LOG_INFO("%s: - device %zu: %s (%s)\n", __func__, i, ggml_backend_dev_name(devs[i]), ggml_backend_dev_description(devs[i]));274            }275 276            GGML_ASSERT(!devs.empty());277            model->get_split_state_ud.n_devices = devs.size();278            model->get_split_state_ud.model     = model;279            gpus.push_back({280                true, ggml_backend_meta_device(281                devs.data(), devs.size(), llama_meta_device_get_split_state, &model->get_split_state_ud)282            });283        } else {284            for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {285                ggml_backend_dev_t dev = ggml_backend_dev_get(i);286                switch (ggml_backend_dev_type(dev)) {287                    case GGML_BACKEND_DEVICE_TYPE_CPU:288                    case GGML_BACKEND_DEVICE_TYPE_ACCEL:289                        // skip CPU backends since they are handled separately290                        break;291 292                    case GGML_BACKEND_DEVICE_TYPE_GPU: {293                        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);294                        if (ggml_backend_reg_name(reg) == std::string("RPC")) {295                            rpc_servers.push_back({false, dev});296                        } else {297                            // check if there is already a GPU with the same device id298                            ggml_backend_dev_props props;299                            ggml_backend_dev_get_props(dev, &props);300                            auto it = std::find_if(gpus.begin(), gpus.end(), [&props](const llama_device & d) {301                                ggml_backend_dev_props d_props;302                                ggml_backend_dev_get_props(d.dev, &d_props);303                                if (props.device_id && d_props.device_id) {304                                    return strcmp(props.device_id, d_props.device_id) == 0;305                                }306                                return false;307                            });308 309                            if (it != gpus.end()) {310                                LLAMA_LOG_INFO("%s: skipping device %s (%s) with id %s - already using device %s (%s) with the same id\n",311                                        __func__,312                                        ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),313                                        props.device_id ? props.device_id : "unknown id",314                                        ggml_backend_dev_name(it->dev), ggml_backend_dev_description(it->dev));315                            } else {316                                gpus.push_back({false, dev});317                            }318                        }319                        break;320                    }321 322                    case GGML_BACKEND_DEVICE_TYPE_IGPU:323                        igpus.push_back({false, dev});324                        break;325                    case GGML_BACKEND_DEVICE_TYPE_META:326                        GGML_ABORT("fatal error");327                }328            }329        }330 331        // add RPC servers at the front of the list to minimize network transfers332        model->devices.insert(model->devices.begin(), rpc_servers.begin(), rpc_servers.end());333 334        // add GPUs335        model->devices.insert(model->devices.end(), gpus.begin(), gpus.end());336 337        // add integrated GPUs only if no other devices were found338        if (model->devices.empty()) {339            model->devices.insert(model->devices.end(), igpus.begin(), igpus.end());340        }341    }342 343    // if using single GPU mode, remove all except the main GPU344    if (params.split_mode == LLAMA_SPLIT_MODE_NONE) {345        if (params.main_gpu < 0) {346            model->devices.clear();347        } else {348            if (params.main_gpu >= (int)model->devices.size()) {349                LLAMA_LOG_ERROR("%s: invalid value for main_gpu: %d (available devices: %zu)\n", __func__, params.main_gpu, model->devices.size());350                llama_model_free(model);351                return nullptr;352            }353            llama_device main_gpu = model->devices[params.main_gpu];354            model->devices.clear();355            model->devices.push_back(main_gpu);356        }357    }358 359    for (const auto & dev : model->devices) {360        ggml_backend_dev_props props;361        ggml_backend_dev_get_props(dev.dev, &props);362        LLAMA_LOG_INFO("%s: using device %s (%s) (%s) - %zu MiB free\n", __func__,363                ggml_backend_dev_name(dev.dev), ggml_backend_dev_description(dev.dev),364                props.device_id ? props.device_id : "unknown id",365                props.memory_free/1024/1024);366    }367 368    const int status = llama_model_load(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, file, *model, params);369    GGML_ASSERT(status <= 0);370    if (status < 0) {371        if (status == -1) {372            LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);373        } else if (status == -2) {374            LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);375        }376 377        llama_model_free(model);378        return nullptr;379    }380 381    return model;382}383 384struct llama_model * llama_model_init_from_user(385        struct gguf_context * metadata,386        llama_model_set_tensor_data_t set_tensor_data,387        void * set_tensor_data_ud,388        struct llama_model_params params) {389    GGML_ASSERT(metadata != nullptr);390    std::string path_model;391    std::vector<std::string> splits = {};392    params.use_mmap = false;393    params.use_extra_bufts = false;394    return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params);395}396// deprecated397struct llama_model * llama_load_model_from_file(398        const char * path_model,399        struct llama_model_params params) {400    return llama_model_load_from_file(path_model, params);401}402 403struct llama_model * llama_model_load_from_file(404        const char * path_model,405        struct llama_model_params params) {406    std::vector<std::string> splits = {};407    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, /*file*/ nullptr, params);408}409 410struct llama_model * llama_model_load_from_splits(411        const char ** paths,412        size_t n_paths,413        struct llama_model_params params) {414    std::vector<std::string> splits;415    if (n_paths == 0) {416        LLAMA_LOG_ERROR("%s: list of splits is empty\n", __func__);417        return nullptr;418    }419    splits.reserve(n_paths);420    for (size_t i = 0; i < n_paths; ++i) {421        splits.push_back(paths[i]);422    }423    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, splits.front(), splits, /*file*/ nullptr, params);424}425 426struct llama_model * llama_model_load_from_file_ptr(FILE * file, struct llama_model_params params) {427    if (!file) {428        LLAMA_LOG_ERROR("%s: file is NULL\n", __func__);429        return nullptr;430    }431    std::string path_model;432    std::vector<std::string> splits = {};433    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, file, params);434}435 436void llama_model_save_to_file(const struct llama_model * model, const char * path_model) {437    llama_model_saver ms(model);438    ms.add_kv_from_model();439    ms.add_tensors_from_model();440    ms.save(path_model);441}442 443//444// chat templates445//446 447int32_t llama_chat_apply_template(448                              const char * tmpl,449         const struct llama_chat_message * chat,450                                  size_t   n_msg,451                                    bool   add_ass,452                                    char * buf,453                                 int32_t   length) {454    const std::string curr_tmpl(tmpl == nullptr ? "chatml" : tmpl);455 456    // format the chat to string457    std::vector<const llama_chat_message *> chat_vec;458    chat_vec.resize(n_msg);459    for (size_t i = 0; i < n_msg; i++) {460        chat_vec[i] = &chat[i];461    }462 463    std::string formatted_chat;464    llm_chat_template detected_tmpl = llm_chat_detect_template(curr_tmpl);465    if (detected_tmpl == LLM_CHAT_TEMPLATE_UNKNOWN) {466        return -1;467    }468    int32_t res = llm_chat_apply_template(detected_tmpl, chat_vec, formatted_chat, add_ass);469    if (res < 0) {470        return res;471    }472    if (buf && length > 0) {473        strncpy(buf, formatted_chat.c_str(), length);474    }475    return res;476}477 478//479// model split480//481 482int32_t llama_split_path(483    char * split_path,484    size_t maxlen,485    const char * path_prefix,486    int32_t split_no,487    int32_t split_count) {488 489    static const char * const SPLIT_PATH_FORMAT = "%s-%05d-of-%05d.gguf";490 491    const int written = snprintf(492        split_path,493        maxlen,494        SPLIT_PATH_FORMAT,495        path_prefix,496        split_no + 1,497        split_count498    );499 500    if (written < 0 || (size_t) written >= maxlen) {501        return 0;502    }503 504    return (int32_t) written;505}506 507int32_t llama_split_prefix(508    char * split_prefix,509    size_t maxlen,510    const char * split_path,511    int32_t split_no,512    int32_t split_count) {513 514    const std::string str_split_path(split_path);515 516    char postfix[32];517    snprintf(postfix, sizeof(postfix), "-%05d-of-%05d.gguf", split_no + 1, split_count);518 519    const std::string str_postfix(postfix);520    if (str_split_path.size() <= str_postfix.size()) {521        return 0;522    }523 524    const size_t size_prefix = str_split_path.size() - str_postfix.size();525 526    if (str_split_path.compare(size_prefix, std::string::npos, str_postfix) == 0) {527        const size_t copy_len = std::min(size_prefix + 1, maxlen);528        snprintf(split_prefix, copy_len, "%s", split_path);529 530        return (int32_t) size_prefix;531    }532 533    return 0;534}535 536const char * llama_print_system_info(void) {537    static std::string s;538    s.clear(); // Clear the string, since it's static, otherwise it will accumulate data from previous calls.539 540    for (size_t i = 0; i < ggml_backend_reg_count(); i++) {541        auto * reg = ggml_backend_reg_get(i);542        auto * get_features_fn = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");543        if (get_features_fn) {544            ggml_backend_feature * features = get_features_fn(reg);545            s += ggml_backend_reg_name(reg);546            s += " : ";547            for (; features->name; features++) {548                s += features->name;549                s += " = ";550                s += features->value;551                s += " | ";552            }553        }554    }555 556    return s.c_str();557}558 559