KBaba7/llama.cpp
0
1#if defined(_MSC_VER)2#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING3#endif4 5#include "ggml.h"6#include "gguf.h"7 8#include "common.h"9#include "log.h"10// Change JSON_ASSERT from assert() to GGML_ASSERT:11#define JSON_ASSERT GGML_ASSERT12#include "json.hpp"13#include "json-schema-to-grammar.h"14#include "llama.h"15#include "chat.hpp"16#include "chat-template.hpp"17 18#include <algorithm>19#include <cinttypes>20#include <climits>21#include <cmath>22#include <codecvt>23#include <cstdarg>24#include <cstring>25#include <ctime>26#include <filesystem>27#include <fstream>28#include <iostream>29#include <iterator>30#include <regex>31#include <sstream>32#include <string>33#include <thread>34#include <unordered_map>35#include <unordered_set>36#include <vector>37 38#if defined(__APPLE__) && defined(__MACH__)39#include <sys/types.h>40#include <sys/sysctl.h>41#endif42 43#if defined(_WIN32)44#define WIN32_LEAN_AND_MEAN45#ifndef NOMINMAX46# define NOMINMAX47#endif48#include <locale>49#include <windows.h>50#include <fcntl.h>51#include <io.h>52#else53#include <sys/ioctl.h>54#include <sys/stat.h>55#include <unistd.h>56#endif57#if defined(LLAMA_USE_CURL)58#include <curl/curl.h>59#include <curl/easy.h>60#include <future>61#endif62 63#if defined(_MSC_VER)64#pragma warning(disable: 4244 4267) // possible loss of data65#endif66 67#if defined(LLAMA_USE_CURL)68#ifdef __linux__69#include <linux/limits.h>70#elif defined(_WIN32)71# if !defined(PATH_MAX)72# define PATH_MAX MAX_PATH73# endif74#else75#include <sys/syslimits.h>76#endif77#define LLAMA_CURL_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 208378 79//80// CURL utils81//82 83using curl_ptr = std::unique_ptr<CURL, decltype(&curl_easy_cleanup)>;84 85// cannot use unique_ptr for curl_slist, because we cannot update without destroying the old one86struct curl_slist_ptr {87 struct curl_slist * ptr = nullptr;88 ~curl_slist_ptr() {89 if (ptr) {90 curl_slist_free_all(ptr);91 }92 }93};94#endif // LLAMA_USE_CURL95 96using json = nlohmann::ordered_json;97 98//99// CPU utils100//101 102int32_t cpu_get_num_physical_cores() {103#ifdef __linux__104 // enumerate the set of thread siblings, num entries is num cores105 std::unordered_set<std::string> siblings;106 for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {107 std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"108 + std::to_string(cpu) + "/topology/thread_siblings");109 if (!thread_siblings.is_open()) {110 break; // no more cpus111 }112 std::string line;113 if (std::getline(thread_siblings, line)) {114 siblings.insert(line);115 }116 }117 if (!siblings.empty()) {118 return static_cast<int32_t>(siblings.size());119 }120#elif defined(__APPLE__) && defined(__MACH__)121 int32_t num_physical_cores;122 size_t len = sizeof(num_physical_cores);123 int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);124 if (result == 0) {125 return num_physical_cores;126 }127 result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);128 if (result == 0) {129 return num_physical_cores;130 }131#elif defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later132 // TODO: windows + arm64 + mingw64133 unsigned int n_threads_win = std::thread::hardware_concurrency();134 unsigned int default_threads = n_threads_win > 0 ? (n_threads_win <= 4 ? n_threads_win : n_threads_win / 2) : 4;135 136 DWORD buffer_size = 0;137 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &buffer_size)) {138 if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {139 return default_threads;140 }141 }142 143 std::vector<char> buffer(buffer_size);144 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data()), &buffer_size)) {145 return default_threads;146 }147 148 int32_t num_physical_cores = 0;149 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data());150 while (buffer_size > 0) {151 if (info->Relationship == RelationProcessorCore) {152 num_physical_cores += info->Processor.GroupCount;153 }154 buffer_size -= info->Size;155 info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(reinterpret_cast<char*>(info) + info->Size);156 }157 158 return num_physical_cores > 0 ? num_physical_cores : default_threads;159#endif160 unsigned int n_threads = std::thread::hardware_concurrency();161 return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;162}163 164#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)165#include <pthread.h>166 167static void cpuid(unsigned leaf, unsigned subleaf,168 unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {169 __asm__("movq\t%%rbx,%%rsi\n\t"170 "cpuid\n\t"171 "xchgq\t%%rbx,%%rsi"172 : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)173 : "0"(leaf), "2"(subleaf));174}175 176static int pin_cpu(int cpu) {177 cpu_set_t mask;178 CPU_ZERO(&mask);179 CPU_SET(cpu, &mask);180 return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);181}182 183static bool is_hybrid_cpu(void) {184 unsigned eax, ebx, ecx, edx;185 cpuid(7, 0, &eax, &ebx, &ecx, &edx);186 return !!(edx & (1u << 15));187}188 189static bool is_running_on_efficiency_core(void) {190 unsigned eax, ebx, ecx, edx;191 cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);192 int intel_atom = 0x20;193 int core_type = (eax & 0xff000000u) >> 24;194 return core_type == intel_atom;195}196 197static int cpu_count_math_cpus(int n_cpu) {198 int result = 0;199 for (int cpu = 0; cpu < n_cpu; ++cpu) {200 if (pin_cpu(cpu)) {201 return -1;202 }203 if (is_running_on_efficiency_core()) {204 continue; // efficiency cores harm lockstep threading205 }206 ++cpu; // hyperthreading isn't useful for linear algebra207 ++result;208 }209 return result;210}211 212#endif // __x86_64__ && __linux__213 214/**215 * Returns number of CPUs on system that are useful for math.216 */217int32_t cpu_get_num_math() {218#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)219 int n_cpu = sysconf(_SC_NPROCESSORS_ONLN);220 if (n_cpu < 1) {221 return cpu_get_num_physical_cores();222 }223 if (is_hybrid_cpu()) {224 cpu_set_t affinity;225 if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {226 int result = cpu_count_math_cpus(n_cpu);227 pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);228 if (result > 0) {229 return result;230 }231 }232 }233#endif234 return cpu_get_num_physical_cores();235}236 237// Helper for setting process priority238 239#if defined(_WIN32)240 241bool set_process_priority(enum ggml_sched_priority prio) {242 if (prio == GGML_SCHED_PRIO_NORMAL) {243 return true;244 }245 246 DWORD p = NORMAL_PRIORITY_CLASS;247 switch (prio) {248 case GGML_SCHED_PRIO_NORMAL: p = NORMAL_PRIORITY_CLASS; break;249 case GGML_SCHED_PRIO_MEDIUM: p = ABOVE_NORMAL_PRIORITY_CLASS; break;250 case GGML_SCHED_PRIO_HIGH: p = HIGH_PRIORITY_CLASS; break;251 case GGML_SCHED_PRIO_REALTIME: p = REALTIME_PRIORITY_CLASS; break;252 }253 254 if (!SetPriorityClass(GetCurrentProcess(), p)) {255 LOG_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError());256 return false;257 }258 259 return true;260}261 262#else // MacOS and POSIX263#include <sys/types.h>264#include <sys/resource.h>265 266bool set_process_priority(enum ggml_sched_priority prio) {267 if (prio == GGML_SCHED_PRIO_NORMAL) {268 return true;269 }270 271 int p = 0;272 switch (prio) {273 case GGML_SCHED_PRIO_NORMAL: p = 0; break;274 case GGML_SCHED_PRIO_MEDIUM: p = -5; break;275 case GGML_SCHED_PRIO_HIGH: p = -10; break;276 case GGML_SCHED_PRIO_REALTIME: p = -20; break;277 }278 279 if (!setpriority(PRIO_PROCESS, 0, p)) {280 LOG_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno);281 return false;282 }283 return true;284}285 286#endif287 288//289// CLI argument parsing290//291 292 293void postprocess_cpu_params(cpu_params& cpuparams, const cpu_params* role_model) {294 int32_t n_set = 0;295 296 if (cpuparams.n_threads < 0) {297 // Assuming everything about cpuparams is invalid298 if (role_model != nullptr) {299 cpuparams = *role_model;300 } else {301 cpuparams.n_threads = cpu_get_num_math();302 }303 }304 305 for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {306 if (cpuparams.cpumask[i]) {307 n_set++;308 }309 }310 311 if (n_set && n_set < cpuparams.n_threads) {312 // Not enough set bits, may experience performance issues.313 LOG_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads);314 }315}316 317bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) {318 size_t dash_loc = range.find('-');319 if (dash_loc == std::string::npos) {320 LOG_ERR("Format of CPU range is invalid! Expected [<start>]-[<end>].\n");321 return false;322 }323 324 size_t start_i;325 size_t end_i;326 327 if (dash_loc == 0) {328 start_i = 0;329 } else {330 start_i = std::stoull(range.substr(0, dash_loc));331 if (start_i >= GGML_MAX_N_THREADS) {332 LOG_ERR("Start index out of bounds!\n");333 return false;334 }335 }336 337 if (dash_loc == range.length() - 1) {338 end_i = GGML_MAX_N_THREADS - 1;339 } else {340 end_i = std::stoull(range.substr(dash_loc + 1));341 if (end_i >= GGML_MAX_N_THREADS) {342 LOG_ERR("End index out of bounds!\n");343 return false;344 }345 }346 347 for (size_t i = start_i; i <= end_i; i++) {348 boolmask[i] = true;349 }350 351 return true;352}353 354bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREADS]) {355 // Discard potential 0x prefix356 size_t start_i = 0;357 if (mask.length() >= 2 && mask.substr(0, 2) == "0x") {358 start_i = 2;359 }360 361 size_t num_digits = mask.length() - start_i;362 if (num_digits > 128) num_digits = 128;363 364 size_t end_i = num_digits + start_i;365 366 for (size_t i = start_i, n = (num_digits*4 - 1); i < end_i; i++, n-=4) {367 char c = mask.at(i);368 int8_t id = c;369 370 if ((c >= '0' && c <= '9')) {371 id -= '0';372 } else if (c >= 'a' && c <= 'f') {373 id -= 'a' - 10;374 } else if (c >= 'A' && c <= 'F') {375 id -= 'A' - 10;376 } else {377 LOG_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i));378 return false;379 }380 381 boolmask[ n ] = boolmask[ n ] || ((id & 8) != 0);382 boolmask[n - 1] = boolmask[n - 1] || ((id & 4) != 0);383 boolmask[n - 2] = boolmask[n - 2] || ((id & 2) != 0);384 boolmask[n - 3] = boolmask[n - 3] || ((id & 1) != 0);385 }386 387 return true;388}389 390void common_init() {391 llama_log_set([](ggml_log_level level, const char * text, void * /*user_data*/) {392 if (LOG_DEFAULT_LLAMA <= common_log_verbosity_thold) {393 common_log_add(common_log_main(), level, "%s", text);394 }395 }, NULL);396 397#ifdef NDEBUG398 const char * build_type = "";399#else400 const char * build_type = " (debug)";401#endif402 403 LOG_INF("build: %d (%s) with %s for %s%s\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT, LLAMA_COMPILER, LLAMA_BUILD_TARGET, build_type);404}405 406std::string common_params_get_system_info(const common_params & params) {407 std::ostringstream os;408 409 os << "system_info: n_threads = " << params.cpuparams.n_threads;410 if (params.cpuparams_batch.n_threads != -1) {411 os << " (n_threads_batch = " << params.cpuparams_batch.n_threads << ")";412 }413#if defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later414 // TODO: windows + arm64 + mingw64415 DWORD logicalProcessorCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);416 os << " / " << logicalProcessorCount << " | " << llama_print_system_info();417#else418 os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();419#endif420 421 return os.str();422}423 424//425// String utils426//427 428std::string string_format(const char * fmt, ...) {429 va_list ap;430 va_list ap2;431 va_start(ap, fmt);432 va_copy(ap2, ap);433 int size = vsnprintf(NULL, 0, fmt, ap);434 GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT435 std::vector<char> buf(size + 1);436 int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);437 GGML_ASSERT(size2 == size);438 va_end(ap2);439 va_end(ap);440 return std::string(buf.data(), size);441}442 443std::string string_strip(const std::string & str) {444 size_t start = 0;445 size_t end = str.size();446 while (start < end && std::isspace(str[start])) {447 start++;448 }449 while (end > start && std::isspace(str[end - 1])) {450 end--;451 }452 return str.substr(start, end - start);453}454 455std::string string_get_sortable_timestamp() {456 using clock = std::chrono::system_clock;457 458 const clock::time_point current_time = clock::now();459 const time_t as_time_t = clock::to_time_t(current_time);460 char timestamp_no_ns[100];461 std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t));462 463 const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(464 current_time.time_since_epoch() % 1000000000).count();465 char timestamp_ns[11];466 snprintf(timestamp_ns, 11, "%09" PRId64, ns);467 468 return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns);469}470 471void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {472 if (search.empty()) {473 return;474 }475 std::string builder;476 builder.reserve(s.length());477 size_t pos = 0;478 size_t last_pos = 0;479 while ((pos = s.find(search, last_pos)) != std::string::npos) {480 builder.append(s, last_pos, pos - last_pos);481 builder.append(replace);482 last_pos = pos + search.length();483 }484 builder.append(s, last_pos, std::string::npos);485 s = std::move(builder);486}487 488std::string string_join(const std::vector<std::string> & values, const std::string & separator) {489 std::ostringstream result;490 for (size_t i = 0; i < values.size(); ++i) {491 if (i > 0) {492 result << separator;493 }494 result << values[i];495 }496 return result.str();497}498 499std::vector<std::string> string_split(const std::string & str, const std::string & delimiter) {500 std::vector<std::string> parts;501 size_t start = 0;502 size_t end = str.find(delimiter);503 504 while (end != std::string::npos) {505 parts.push_back(str.substr(start, end - start));506 start = end + delimiter.length();507 end = str.find(delimiter, start);508 }509 510 parts.push_back(str.substr(start));511 512 return parts;513}514 515std::string string_repeat(const std::string & str, size_t n) {516 if (n == 0) {517 return "";518 }519 520 std::string result;521 result.reserve(str.length() * n);522 523 for (size_t i = 0; i < n; ++i) {524 result += str;525 }526 527 return result;528}529 530std::string string_from(bool value) {531 return value ? "true" : "false";532}533 534std::string string_from(const std::vector<int> & values) {535 std::stringstream buf;536 537 buf << "[ ";538 bool first = true;539 for (auto e : values) {540 if (first) {541 first = false;542 } else {543 buf << ", ";544 }545 buf << std::to_string(e);546 }547 buf << " ]";548 549 return buf.str();550}551 552std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens) {553 std::stringstream buf;554 555 buf << "[ ";556 557 bool first = true;558 for (const auto & token : tokens) {559 if (!first) {560 buf << ", ";561 } else {562 first = false;563 }564 565 auto detokenized = common_token_to_piece(ctx, token);566 567 detokenized.erase(568 std::remove_if(569 detokenized.begin(),570 detokenized.end(),571 [](const unsigned char c) { return !std::isprint(c); }),572 detokenized.end());573 574 buf << "'" << detokenized << "'"575 << ":" << std::to_string(token);576 }577 578 buf << " ]";579 580 return buf.str();581}582 583std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {584 std::stringstream buf;585 586 buf << "[ ";587 588 bool first = true;589 for (int i = 0; i < batch.n_tokens; ++i) {590 if (!first) {591 buf << ", ";592 } else {593 first = false;594 }595 596 auto detokenized = common_token_to_piece(ctx, batch.token[i]);597 598 detokenized.erase(599 std::remove_if(600 detokenized.begin(),601 detokenized.end(),602 [](const unsigned char c) { return !std::isprint(c); }),603 detokenized.end());604 605 buf << "\n" << std::to_string(i)606 << ", token '" << detokenized << "'"607 << ", pos " << std::to_string(batch.pos[i])608 << ", n_seq_id " << std::to_string(batch.n_seq_id[i])609 << ", seq_id " << std::to_string(batch.seq_id[i][0])610 << ", logits " << std::to_string(batch.logits[i]);611 }612 613 buf << " ]";614 615 return buf.str();616}617 618void string_process_escapes(std::string & input) {619 std::size_t input_len = input.length();620 std::size_t output_idx = 0;621 622 for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) {623 if (input[input_idx] == '\\' && input_idx + 1 < input_len) {624 switch (input[++input_idx]) {625 case 'n': input[output_idx++] = '\n'; break;626 case 'r': input[output_idx++] = '\r'; break;627 case 't': input[output_idx++] = '\t'; break;628 case '\'': input[output_idx++] = '\''; break;629 case '\"': input[output_idx++] = '\"'; break;630 case '\\': input[output_idx++] = '\\'; break;631 case 'x':632 // Handle \x12, etc633 if (input_idx + 2 < input_len) {634 const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 };635 char *err_p = nullptr;636 const long val = std::strtol(x, &err_p, 16);637 if (err_p == x + 2) {638 input_idx += 2;639 input[output_idx++] = char(val);640 break;641 }642 }643 // fall through644 default: input[output_idx++] = '\\';645 input[output_idx++] = input[input_idx]; break;646 }647 } else {648 input[output_idx++] = input[input_idx];649 }650 }651 652 input.resize(output_idx);653}654 655bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {656 const char * sep = strchr(data, '=');657 if (sep == nullptr || sep - data >= 128) {658 LOG_ERR("%s: malformed KV override '%s'\n", __func__, data);659 return false;660 }661 llama_model_kv_override kvo;662 std::strncpy(kvo.key, data, sep - data);663 kvo.key[sep - data] = 0;664 sep++;665 if (strncmp(sep, "int:", 4) == 0) {666 sep += 4;667 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;668 kvo.val_i64 = std::atol(sep);669 } else if (strncmp(sep, "float:", 6) == 0) {670 sep += 6;671 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;672 kvo.val_f64 = std::atof(sep);673 } else if (strncmp(sep, "bool:", 5) == 0) {674 sep += 5;675 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;676 if (std::strcmp(sep, "true") == 0) {677 kvo.val_bool = true;678 } else if (std::strcmp(sep, "false") == 0) {679 kvo.val_bool = false;680 } else {681 LOG_ERR("%s: invalid boolean value for KV override '%s'\n", __func__, data);682 return false;683 }684 } else if (strncmp(sep, "str:", 4) == 0) {685 sep += 4;686 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;687 if (strlen(sep) > 127) {688 LOG_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);689 return false;690 }691 strncpy(kvo.val_str, sep, 127);692 kvo.val_str[127] = '\0';693 } else {694 LOG_ERR("%s: invalid type for KV override '%s'\n", __func__, data);695 return false;696 }697 overrides.emplace_back(std::move(kvo));698 return true;699}700 701//702// Filesystem utils703//704 705// Validate if a filename is safe to use706// To validate a full path, split the path by the OS-specific path separator, and validate each part with this function707bool fs_validate_filename(const std::string & filename) {708 if (!filename.length()) {709 // Empty filename invalid710 return false;711 }712 if (filename.length() > 255) {713 // Limit at common largest possible filename on Linux filesystems714 // to avoid unnecessary further validation715 // (On systems with smaller limits it will be caught by the OS)716 return false;717 }718 719 std::u32string filename_utf32;720 try {721#if defined(__clang__)722 // disable C++17 deprecation warning for std::codecvt_utf8723# pragma clang diagnostic push724# pragma clang diagnostic ignored "-Wdeprecated-declarations"725#endif726 std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;727 728#if defined(__clang__)729# pragma clang diagnostic pop730#endif731 732 filename_utf32 = converter.from_bytes(filename);733 734 // If the reverse conversion mismatches, it means overlong UTF-8 sequences were used,735 // or invalid encodings were encountered. Reject such attempts736 std::string filename_reencoded = converter.to_bytes(filename_utf32);737 if (filename_reencoded != filename) {738 return false;739 }740 } catch (const std::exception &) {741 return false;742 }743 744 // Check for forbidden codepoints:745 // - Control characters746 // - Unicode equivalents of illegal characters747 // - UTF-16 surrogate pairs748 // - UTF-8 replacement character749 // - Byte order mark (BOM)750 // - Illegal characters: / \ : * ? " < > |751 for (char32_t c : filename_utf32) {752 if (c <= 0x1F // Control characters (C0)753 || c == 0x7F // Control characters (DEL)754 || (c >= 0x80 && c <= 0x9F) // Control characters (C1)755 || c == 0xFF0E // Fullwidth Full Stop (period equivalent)756 || c == 0x2215 // Division Slash (forward slash equivalent)757 || c == 0x2216 // Set Minus (backslash equivalent)758 || (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs759 || c == 0xFFFD // Replacement Character (UTF-8)760 || c == 0xFEFF // Byte Order Mark (BOM)761 || c == '/' || c == '\\' || c == ':' || c == '*' // Illegal characters762 || c == '?' || c == '"' || c == '<' || c == '>' || c == '|') {763 return false;764 }765 }766 767 // Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename768 // Unicode and other whitespace is not affected, only 0x20 space769 if (filename.front() == ' ' || filename.back() == ' ' || filename.back() == '.') {770 return false;771 }772 773 // Reject any ".." (currently stricter than necessary, it should be fine to just check for == ".." instead)774 if (filename.find("..") != std::string::npos) {775 return false;776 }777 778 // Reject "."779 if (filename == ".") {780 return false;781 }782 783 return true;784}785 786// returns true if successful, false otherwise787bool fs_create_directory_with_parents(const std::string & path) {788#ifdef _WIN32789 std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;790 std::wstring wpath = converter.from_bytes(path);791 792 // if the path already exists, check whether it's a directory793 const DWORD attributes = GetFileAttributesW(wpath.c_str());794 if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {795 return true;796 }797 798 size_t pos_slash = 0;799 800 // process path from front to back, procedurally creating directories801 while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {802 const std::wstring subpath = wpath.substr(0, pos_slash);803 const wchar_t * test = subpath.c_str();804 805 const bool success = CreateDirectoryW(test, NULL);806 if (!success) {807 const DWORD error = GetLastError();808 809 // if the path already exists, ensure that it's a directory810 if (error == ERROR_ALREADY_EXISTS) {811 const DWORD attributes = GetFileAttributesW(subpath.c_str());812 if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {813 return false;814 }815 } else {816 return false;817 }818 }819 820 pos_slash += 1;821 }822 823 return true;824#else825 // if the path already exists, check whether it's a directory826 struct stat info;827 if (stat(path.c_str(), &info) == 0) {828 return S_ISDIR(info.st_mode);829 }830 831 size_t pos_slash = 1; // skip leading slashes for directory creation832 833 // process path from front to back, procedurally creating directories834 while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {835 const std::string subpath = path.substr(0, pos_slash);836 struct stat info;837 838 // if the path already exists, ensure that it's a directory839 if (stat(subpath.c_str(), &info) == 0) {840 if (!S_ISDIR(info.st_mode)) {841 return false;842 }843 } else {844 // create parent directories845 const int ret = mkdir(subpath.c_str(), 0755);846 if (ret != 0) {847 return false;848 }849 }850 851 pos_slash += 1;852 }853 854 return true;855#endif // _WIN32856}857 858std::string fs_get_cache_directory() {859 std::string cache_directory = "";860 auto ensure_trailing_slash = [](std::string p) {861 // Make sure to add trailing slash862 if (p.back() != DIRECTORY_SEPARATOR) {863 p += DIRECTORY_SEPARATOR;864 }865 return p;866 };867 if (getenv("LLAMA_CACHE")) {868 cache_directory = std::getenv("LLAMA_CACHE");869 } else {870#ifdef __linux__871 if (std::getenv("XDG_CACHE_HOME")) {872 cache_directory = std::getenv("XDG_CACHE_HOME");873 } else {874 cache_directory = std::getenv("HOME") + std::string("/.cache/");875 }876#elif defined(__APPLE__)877 cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");878#elif defined(_WIN32)879 cache_directory = std::getenv("LOCALAPPDATA");880#endif // __linux__881 cache_directory = ensure_trailing_slash(cache_directory);882 cache_directory += "llama.cpp";883 }884 return ensure_trailing_slash(cache_directory);885}886 887std::string fs_get_cache_file(const std::string & filename) {888 GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);889 std::string cache_directory = fs_get_cache_directory();890 const bool success = fs_create_directory_with_parents(cache_directory);891 if (!success) {892 throw std::runtime_error("failed to create cache directory: " + cache_directory);893 }894 return cache_directory + filename;895}896 897 898//899// Model utils900//901struct common_init_result common_init_from_params(common_params & params) {902 common_init_result iparams;903 auto mparams = common_model_params_to_llama(params);904 905 llama_model * model = nullptr;906 907 if (!params.hf_repo.empty() && !params.hf_file.empty()) {908 model = common_load_model_from_hf(params.hf_repo, params.hf_file, params.model, params.hf_token, mparams);909 } else if (!params.model_url.empty()) {910 model = common_load_model_from_url(params.model_url, params.model, params.hf_token, mparams);911 } else {912 model = llama_model_load_from_file(params.model.c_str(), mparams);913 }914 915 if (model == NULL) {916 LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.c_str());917 return iparams;918 }919 920 const llama_vocab * vocab = llama_model_get_vocab(model);921 922 if (params.reranking) {923 bool ok = true;924 925 if (llama_vocab_bos(vocab) == LLAMA_TOKEN_NULL) {926 LOG_WRN("%s: warning: vocab does not have a BOS token, reranking will not work\n", __func__);927 ok = false;928 }929 930 if (llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) {931 LOG_WRN("%s: warning: vocab does not have an EOS token, reranking will not work\n", __func__);932 ok = false;933 }934 935 if (llama_vocab_sep(vocab) == LLAMA_TOKEN_NULL) {936 LOG_WRN("%s: warning: vocab does not have a SEP token, reranking will not work\n", __func__);937 ok = false;938 }939 940 if (!ok) {941 llama_model_free(model);942 943 return iparams;944 }945 }946 947 auto cparams = common_context_params_to_llama(params);948 949 llama_context * lctx = llama_init_from_model(model, cparams);950 if (lctx == NULL) {951 LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.c_str());952 llama_model_free(model);953 return iparams;954 }955 956 if (params.ctx_shift && !llama_kv_cache_can_shift(lctx)) {957 LOG_WRN("%s: KV cache shifting is not supported for this model, disabling KV cache shifting\n", __func__);958 params.ctx_shift = false;959 }960 961 if (!params.control_vectors.empty()) {962 if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;963 if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);964 965 const auto cvec = common_control_vector_load(params.control_vectors);966 if (cvec.n_embd == -1) {967 llama_free(lctx);968 llama_model_free(model);969 970 return iparams;971 }972 973 int err = llama_apply_adapter_cvec(974 lctx,975 cvec.data.data(),976 cvec.data.size(),977 cvec.n_embd,978 params.control_vector_layer_start,979 params.control_vector_layer_end);980 if (err) {981 llama_free(lctx);982 llama_model_free(model);983 984 return iparams;985 }986 }987 988 // load and optionally apply lora adapters989 for (auto & la : params.lora_adapters) {990 llama_adapter_lora_ptr lora;991 lora.reset(llama_adapter_lora_init(model, la.path.c_str()));992 if (lora == nullptr) {993 LOG_ERR("%s: failed to apply lora adapter '%s'\n", __func__, la.path.c_str());994 llama_free(lctx);995 llama_model_free(model);996 return iparams;997 }998 999 la.ptr = lora.get();1000 iparams.lora.emplace_back(std::move(lora)); // copy to list of loaded adapters1001 }1002 1003 if (!params.lora_init_without_apply) {1004 common_set_adapter_lora(lctx, params.lora_adapters);1005 }1006 1007 if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) {1008 LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__);1009 params.sampling.ignore_eos = false;1010 }1011 1012 if (params.sampling.ignore_eos) {1013 for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) {1014 if (llama_vocab_is_eog(vocab, i)) {1015 LOG_INF("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(lctx, i).c_str(), -INFINITY);1016 params.sampling.logit_bias.push_back({i, -INFINITY});1017 }1018 }1019 }1020 1021 if (params.sampling.penalty_last_n == -1) {1022 LOG_INF("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));1023 params.sampling.penalty_last_n = llama_n_ctx(lctx);1024 }1025 1026 if (params.sampling.dry_penalty_last_n == -1) {1027 LOG_INF("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));1028 params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);1029 }1030 1031 if (params.warmup) {1032 LOG_WRN("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__);1033 1034 std::vector<llama_token> tmp;1035 llama_token bos = llama_vocab_bos(vocab);1036 llama_token eos = llama_vocab_eos(vocab);1037 1038 // some models (e.g. T5) don't have a BOS token1039 if (bos != LLAMA_TOKEN_NULL) {1040 tmp.push_back(bos);1041 }1042 if (eos != LLAMA_TOKEN_NULL) {1043 tmp.push_back(eos);1044 }1045 if (tmp.empty()) {1046 tmp.push_back(0);1047 }1048 1049 if (llama_model_has_encoder(model)) {1050 llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));1051 llama_token decoder_start_token_id = llama_model_decoder_start_token(model);1052 if (decoder_start_token_id == LLAMA_TOKEN_NULL) {1053 decoder_start_token_id = bos;1054 }1055 tmp.clear();1056 tmp.push_back(decoder_start_token_id);1057 }1058 if (llama_model_has_decoder(model)) {1059 llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));1060 }1061 llama_kv_cache_clear(lctx);1062 llama_synchronize(lctx);1063 llama_perf_context_reset(lctx);1064 }1065 1066 iparams.model.reset(model);1067 iparams.context.reset(lctx);1068 1069 return iparams;1070}1071 1072void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora) {1073 llama_clear_adapter_lora(ctx);1074 for (auto & la : lora) {1075 if (la.scale != 0.0f) {1076 llama_set_adapter_lora(ctx, la.ptr, la.scale);1077 }1078 }1079}1080 1081struct llama_model_params common_model_params_to_llama(common_params & params) {1082 auto mparams = llama_model_default_params();1083 1084 if (!params.devices.empty()) {1085 mparams.devices = params.devices.data();1086 }1087 if (params.n_gpu_layers != -1) {1088 mparams.n_gpu_layers = params.n_gpu_layers;1089 }1090 mparams.main_gpu = params.main_gpu;1091 mparams.split_mode = params.split_mode;1092 mparams.tensor_split = params.tensor_split;1093 mparams.use_mmap = params.use_mmap;1094 mparams.use_mlock = params.use_mlock;1095 mparams.check_tensors = params.check_tensors;1096 if (params.kv_overrides.empty()) {1097 mparams.kv_overrides = NULL;1098 } else {1099 GGML_ASSERT(params.kv_overrides.back().key[0] == 0 && "KV overrides not terminated with empty key");1100 mparams.kv_overrides = params.kv_overrides.data();1101 }1102 1103 return mparams;1104}1105 1106struct llama_context_params common_context_params_to_llama(const common_params & params) {1107 auto cparams = llama_context_default_params();1108 1109 cparams.n_ctx = params.n_ctx;1110 cparams.n_seq_max = params.n_parallel;1111 cparams.n_batch = params.n_batch;1112 cparams.n_ubatch = params.n_ubatch;1113 cparams.n_threads = params.cpuparams.n_threads;1114 cparams.n_threads_batch = params.cpuparams_batch.n_threads == -1 ?1115 params.cpuparams.n_threads : params.cpuparams_batch.n_threads;1116 cparams.logits_all = params.logits_all;1117 cparams.embeddings = params.embedding;1118 cparams.rope_scaling_type = params.rope_scaling_type;1119 cparams.rope_freq_base = params.rope_freq_base;1120 cparams.rope_freq_scale = params.rope_freq_scale;1121 cparams.yarn_ext_factor = params.yarn_ext_factor;1122 cparams.yarn_attn_factor = params.yarn_attn_factor;1123 cparams.yarn_beta_fast = params.yarn_beta_fast;1124 cparams.yarn_beta_slow = params.yarn_beta_slow;1125 cparams.yarn_orig_ctx = params.yarn_orig_ctx;1126 cparams.pooling_type = params.pooling_type;1127 cparams.attention_type = params.attention_type;1128 cparams.defrag_thold = params.defrag_thold;1129 cparams.cb_eval = params.cb_eval;1130 cparams.cb_eval_user_data = params.cb_eval_user_data;1131 cparams.offload_kqv = !params.no_kv_offload;1132 cparams.flash_attn = params.flash_attn;1133 cparams.no_perf = params.no_perf;1134 1135 if (params.reranking) {1136 cparams.embeddings = true;1137 cparams.pooling_type = LLAMA_POOLING_TYPE_RANK;1138 }1139 1140 cparams.type_k = params.cache_type_k;1141 cparams.type_v = params.cache_type_v;1142 1143 return cparams;1144}1145 1146struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const cpu_params & params) {1147 struct ggml_threadpool_params tpp;1148 1149 ggml_threadpool_params_init(&tpp, params.n_threads); // setup the defaults1150 1151 if (params.mask_valid) {1152 std::memcpy(&tpp.cpumask, ¶ms.cpumask, GGML_MAX_N_THREADS);1153 }1154 1155 tpp.prio = params.priority;1156 tpp.poll = params.poll;1157 tpp.strict_cpu = params.strict_cpu;1158 1159 return tpp;1160}1161 1162#ifdef LLAMA_USE_CURL1163 1164#define CURL_MAX_RETRY 31165#define CURL_RETRY_DELAY_SECONDS 21166 1167static bool curl_perform_with_retry(const std::string & url, CURL * curl, int max_attempts, int retry_delay_seconds) {1168 int remaining_attempts = max_attempts;1169 1170 while (remaining_attempts > 0) {1171 LOG_INF("%s: Trying to download from %s (attempt %d of %d)...\n", __func__ , url.c_str(), max_attempts - remaining_attempts + 1, max_attempts);1172 1173 CURLcode res = curl_easy_perform(curl);1174 if (res == CURLE_OK) {1175 return true;1176 }1177 1178 int exponential_backoff_delay = std::pow(retry_delay_seconds, max_attempts - remaining_attempts) * 1000;1179 LOG_WRN("%s: curl_easy_perform() failed: %s, retrying after %d milliseconds...\n", __func__, curl_easy_strerror(res), exponential_backoff_delay);1180 1181 remaining_attempts--;1182 std::this_thread::sleep_for(std::chrono::milliseconds(exponential_backoff_delay));1183 }1184 1185 LOG_ERR("%s: curl_easy_perform() failed after %d attempts\n", __func__, max_attempts);1186 1187 return false;1188}1189 1190static bool common_download_file(const std::string & url, const std::string & path, const std::string & hf_token) {1191 // Initialize libcurl1192 curl_ptr curl(curl_easy_init(), &curl_easy_cleanup);1193 curl_slist_ptr http_headers;1194 if (!curl) {1195 LOG_ERR("%s: error initializing libcurl\n", __func__);1196 return false;1197 }1198 1199 bool force_download = false;1200