Brunobkr/llama.cpp_AlgMor24_github

1#include "ggml.h"2#include "gguf.h"3 4#include "build-info.h"5#include "common.h"6#include "fit.h"7#include "log.h"8#include "llama.h"9#include "sampling.h"10#include "speculative.h"11#include "unicode.h"12 13#include <algorithm>14#include <cinttypes>15#include <climits>16#include <cmath>17#include <chrono>18#include <cstdarg>19#include <cstring>20#include <ctime>21#include <filesystem>22#include <fstream>23#include <iostream>24#include <iterator>25#include <regex>26#include <sstream>27#include <string>28#include <thread>29#include <unordered_set>30#include <vector>31 32#if defined(__APPLE__) && defined(__MACH__)33#include <sys/types.h>34#include <sys/sysctl.h>35#endif36 37#if defined(_WIN32)38#define WIN32_LEAN_AND_MEAN39#ifndef NOMINMAX40#   define NOMINMAX41#endif42#include <locale>43#include <windows.h>44#include <string.h>45#include <fcntl.h>46#include <io.h>47#else48#include <sys/ioctl.h>49#include <sys/stat.h>50#include <unistd.h>51#endif52 53#if defined(__linux__)54#include <sys/types.h>55#include <pwd.h>56#endif57 58#if defined(_AIX)59#include <sys/systemcfg.h>60#endif61 62#if defined(_MSC_VER)63#pragma warning(disable: 4244 4267) // possible loss of data64#endif65 66common_time_meas::common_time_meas(int64_t & t_acc, bool disable) : t_start_us(disable ? -1 : ggml_time_us()), t_acc(t_acc) {}67 68common_time_meas::~common_time_meas() {69    if (t_start_us >= 0) {70        t_acc += ggml_time_us() - t_start_us;71    }72}73 74//75// CPU utils76//77 78int32_t common_cpu_get_num_physical_cores() {79#if defined(_AIX)80    int32_t logical_cpus = _system_configuration.ncpus;81    int32_t smt_threads = _system_configuration.smt_threads;82    if (smt_threads > 0) {83        return static_cast<int32_t>(logical_cpus / smt_threads);84    }85    if (logical_cpus > 0) {86        return static_cast<int32_t>(logical_cpus);87    }88#elif defined(__linux__)89    // enumerate the set of thread siblings, num entries is num cores90    std::unordered_set<std::string> siblings;91    for (uint32_t cpu=0; cpu < UINT32_MAX; ++cpu) {92        std::ifstream thread_siblings("/sys/devices/system/cpu/cpu"93            + std::to_string(cpu) + "/topology/thread_siblings");94        if (!thread_siblings.is_open()) {95            break; // no more cpus96        }97        std::string line;98        if (std::getline(thread_siblings, line)) {99            siblings.insert(line);100        }101    }102    if (!siblings.empty()) {103        return static_cast<int32_t>(siblings.size());104    }105#elif defined(__APPLE__) && defined(__MACH__)106    int32_t num_physical_cores;107    size_t len = sizeof(num_physical_cores);108    int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);109    if (result == 0) {110        return num_physical_cores;111    }112    result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);113    if (result == 0) {114        return num_physical_cores;115    }116#elif defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later117    // TODO: windows + arm64 + mingw64118    unsigned int n_threads_win = std::thread::hardware_concurrency();119    unsigned int default_threads = n_threads_win > 0 ? (n_threads_win <= 4 ? n_threads_win : n_threads_win / 2) : 4;120 121    DWORD buffer_size = 0;122    if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &buffer_size)) {123        if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {124            return default_threads;125        }126    }127 128    std::vector<char> buffer(buffer_size);129    if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data()), &buffer_size)) {130        return default_threads;131    }132 133    int32_t num_physical_cores = 0;134    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(buffer.data());135    while (buffer_size > 0) {136        if (info->Relationship == RelationProcessorCore) {137            num_physical_cores += info->Processor.GroupCount;138        }139        buffer_size -= info->Size;140        info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>(reinterpret_cast<char*>(info) + info->Size);141    }142 143    return num_physical_cores > 0 ? num_physical_cores : default_threads;144#endif145    unsigned int n_threads = std::thread::hardware_concurrency();146    return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;147}148 149#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)150#include <pthread.h>151 152static void cpuid(unsigned leaf, unsigned subleaf,153                  unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) {154    __asm__("movq\t%%rbx,%%rsi\n\t"155            "cpuid\n\t"156            "xchgq\t%%rbx,%%rsi"157            : "=a"(*eax), "=S"(*ebx), "=c"(*ecx), "=d"(*edx)158            : "0"(leaf), "2"(subleaf));159}160 161static int pin_cpu(int cpu) {162    cpu_set_t mask;163    CPU_ZERO(&mask);164    CPU_SET(cpu, &mask);165    return pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask);166}167 168static bool is_hybrid_cpu(void) {169    unsigned eax, ebx, ecx, edx;170    cpuid(7, 0, &eax, &ebx, &ecx, &edx);171    return !!(edx & (1u << 15));172}173 174static bool is_running_on_efficiency_core(void) {175    unsigned eax, ebx, ecx, edx;176    cpuid(0x1a, 0, &eax, &ebx, &ecx, &edx);177    int intel_atom = 0x20;178    int core_type = (eax & 0xff000000u) >> 24;179    return core_type == intel_atom;180}181 182static int cpu_count_math_cpus(int n_cpu) {183    int result = 0;184    for (int cpu = 0; cpu < n_cpu; ++cpu) {185        if (pin_cpu(cpu)) {186            return -1;187        }188        if (is_running_on_efficiency_core()) {189            continue; // efficiency cores harm lockstep threading190        }191        ++cpu; // hyperthreading isn't useful for linear algebra192        ++result;193    }194    return result;195}196 197#endif // __x86_64__ && __linux__198 199/**200 * Returns number of CPUs on system that are useful for math.201 */202int32_t common_cpu_get_num_math() {203#if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__)204    int n_cpu = sysconf(_SC_NPROCESSORS_ONLN);205    if (n_cpu < 1) {206        return common_cpu_get_num_physical_cores();207    }208    if (is_hybrid_cpu()) {209        cpu_set_t affinity;210        if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) {211            int result = cpu_count_math_cpus(n_cpu);212            pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity);213            if (result > 0) {214                return result;215            }216        }217    }218#elif defined(__powerpc64__) || defined(__powerpc__)219    int32_t smt_factor = 1;220    int phy_cpus = common_cpu_get_num_physical_cores();221    int logical_cpus = sysconf(_SC_NPROCESSORS_ONLN);222    if (phy_cpus > 0 && logical_cpus > phy_cpus) {223        smt_factor = logical_cpus / phy_cpus;224    }225    return phy_cpus * std::min(smt_factor, 2);226#endif227    return common_cpu_get_num_physical_cores();228}229 230// Helper for setting process priority231 232#if defined(_WIN32)233 234bool set_process_priority(enum ggml_sched_priority prio) {235    if (prio == GGML_SCHED_PRIO_NORMAL) {236        return true;237    }238 239    DWORD p = NORMAL_PRIORITY_CLASS;240    switch (prio) {241        case GGML_SCHED_PRIO_LOW:      p = BELOW_NORMAL_PRIORITY_CLASS; break;242        case GGML_SCHED_PRIO_NORMAL:   p = NORMAL_PRIORITY_CLASS;       break;243        case GGML_SCHED_PRIO_MEDIUM:   p = ABOVE_NORMAL_PRIORITY_CLASS; break;244        case GGML_SCHED_PRIO_HIGH:     p = HIGH_PRIORITY_CLASS;         break;245        case GGML_SCHED_PRIO_REALTIME: p = REALTIME_PRIORITY_CLASS;     break;246    }247 248    if (!SetPriorityClass(GetCurrentProcess(), p)) {249        COM_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError());250        return false;251    }252 253    return true;254}255 256#else // MacOS and POSIX257#include <sys/types.h>258#include <sys/resource.h>259 260bool set_process_priority(enum ggml_sched_priority prio) {261    if (prio == GGML_SCHED_PRIO_NORMAL) {262        return true;263    }264 265    int p = 0;266    switch (prio) {267        case GGML_SCHED_PRIO_LOW:      p =  5;  break;268        case GGML_SCHED_PRIO_NORMAL:   p =  0;  break;269        case GGML_SCHED_PRIO_MEDIUM:   p = -5;  break;270        case GGML_SCHED_PRIO_HIGH:     p = -10; break;271        case GGML_SCHED_PRIO_REALTIME: p = -20; break;272    }273 274    if (setpriority(PRIO_PROCESS, 0, p) != 0) {275        COM_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno);276        return false;277    }278    return true;279}280 281#endif282 283//284// CLI argument parsing285//286 287 288void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_params * role_model) {289    int32_t n_set = 0;290 291    if (cpuparams.n_threads < 0) {292        // Assuming everything about cpuparams is invalid293        if (role_model != nullptr) {294            cpuparams = *role_model;295        } else {296            cpuparams.n_threads = common_cpu_get_num_math();297        }298    }299 300    for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {301        if (cpuparams.cpumask[i]) {302            n_set++;303        }304    }305 306    if (n_set && n_set < cpuparams.n_threads) {307        // Not enough set bits, may experience performance issues.308        COM_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads);309    }310}311 312bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) {313    size_t dash_loc = range.find('-');314    if (dash_loc == std::string::npos) {315        COM_ERR("%s", "Format of CPU range is invalid! Expected [<start>]-[<end>].\n");316        return false;317    }318 319    size_t start_i;320    size_t end_i;321 322    if (dash_loc == 0) {323        start_i = 0;324    } else {325        start_i = std::stoull(range.substr(0, dash_loc));326        if (start_i >= GGML_MAX_N_THREADS) {327            COM_ERR("%s", "Start index out of bounds!\n");328            return false;329        }330    }331 332    if (dash_loc == range.length() - 1) {333        end_i = GGML_MAX_N_THREADS - 1;334    } else {335        end_i = std::stoull(range.substr(dash_loc + 1));336        if (end_i >= GGML_MAX_N_THREADS) {337            COM_ERR("%s", "End index out of bounds!\n");338            return false;339        }340    }341 342    for (size_t i = start_i; i <= end_i; i++) {343        boolmask[i] = true;344    }345 346    return true;347}348 349bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREADS]) {350    // Discard potential 0x prefix351    size_t start_i = 0;352    if (mask.length() >= 2 && mask.substr(0, 2) == "0x") {353        start_i = 2;354    }355 356    size_t num_digits = mask.length() - start_i;357    num_digits = std::min<size_t>(num_digits, 128);358 359    size_t end_i = num_digits + start_i;360 361    for (size_t i = start_i, n = (num_digits*4 - 1); i < end_i; i++, n-=4) {362        char c = mask.at(i);363        int8_t id = c;364 365        if ((c >= '0' && c <= '9')) {366            id -= '0';367        } else if (c >= 'a' && c <= 'f') {368            id -= 'a' - 10;369        } else if (c >= 'A' && c <= 'F') {370            id -= 'A' - 10;371        } else {372            COM_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i));373            return false;374        }375 376        boolmask[  n  ] = boolmask[  n  ] || ((id & 8) != 0);377        boolmask[n - 1] = boolmask[n - 1] || ((id & 4) != 0);378        boolmask[n - 2] = boolmask[n - 2] || ((id & 2) != 0);379        boolmask[n - 3] = boolmask[n - 3] || ((id & 1) != 0);380    }381 382    return true;383}384 385void common_init() {386#if defined(_WIN32)387    SetConsoleOutputCP(CP_UTF8);388    SetConsoleCP(CP_UTF8);389#endif390 391    common_log_set_prefix(common_log_main(), true);392    common_log_set_timestamps(common_log_main(), true);393 394    llama_log_set(common_log_default_callback, NULL);395}396 397void common_params_print_info(const common_params & params, bool print_devices) {398#ifdef NDEBUG399    const char * build_type = "";400#else401    const char * build_type = " (debug)";402#endif403    COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);404 405    COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, common_log_get_verbosity_thold());406 407    // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device408    if (print_devices) {409        COM_TRC("%s", "device_info:\n");410        for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {411            auto * dev = ggml_backend_dev_get(i);412            size_t free, total;413            ggml_backend_dev_memory(dev, &free, &total);414            COM_TRC("  - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);415        }416    }417    COM_TRC("%s\n", common_params_get_system_info(params).c_str());418}419 420std::string common_params_get_system_info(const common_params & params) {421    std::ostringstream os;422 423    os << "system_info: n_threads = " << params.cpuparams.n_threads;424    if (params.cpuparams_batch.n_threads != -1) {425        os << " (n_threads_batch = " << params.cpuparams_batch.n_threads << ")";426    }427#if defined(_WIN32) && (_WIN32_WINNT >= 0x0601) && !defined(__MINGW64__) // windows 7 and later428    // TODO: windows + arm64 + mingw64429    DWORD logicalProcessorCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);430    os << " / " << logicalProcessorCount << " | " << llama_print_system_info();431#else432    os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();433#endif434 435    return os.str();436}437 438//439// String utils440//441 442std::string string_format(const char * fmt, ...) {443    va_list ap;444    va_list ap2;445    va_start(ap, fmt);446    va_copy(ap2, ap);447    int size = vsnprintf(NULL, 0, fmt, ap);448    GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT449    std::vector<char> buf(size + 1);450    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);451    GGML_ASSERT(size2 == size);452    va_end(ap2);453    va_end(ap);454    return std::string(buf.data(), size);455}456 457std::string string_strip(const std::string & str) {458    size_t start = 0;459    size_t end = str.size();460    while (start < end && std::isspace(str[start])) {461        start++;462    }463    while (end > start && std::isspace(str[end - 1])) {464        end--;465    }466    return str.substr(start, end - start);467}468 469std::string string_lcs(std::string_view a, std::string_view b) {470    if (a.empty() || b.empty()) return {};471 472    std::vector<std::vector<size_t>> dp(a.size() + 1, std::vector<size_t>(b.size() + 1, 0));473    size_t best_len = 0;474    size_t best_end_a = 0;475 476    for (size_t i = 1; i <= a.size(); ++i) {477        for (size_t j = 1; j <= b.size(); ++j) {478            if (a[i - 1] == b[j - 1]) {479                dp[i][j] = dp[i - 1][j - 1] + 1;480                if (dp[i][j] > best_len) {481                    best_len = dp[i][j];482                    best_end_a = i;483                }484            }485        }486    }487    return std::string(a.substr(best_end_a - best_len, best_len));488}489 490std::string string_get_sortable_timestamp() {491    using clock = std::chrono::system_clock;492 493    const clock::time_point current_time = clock::now();494    const time_t as_time_t = clock::to_time_t(current_time);495    char timestamp_no_ns[100];496    std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t));497 498    const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(499        current_time.time_since_epoch() % 1000000000).count();500    char timestamp_ns[11];501    snprintf(timestamp_ns, 11, "%09" PRId64, ns);502 503    return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns);504}505 506void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {507    if (search.empty()) {508        return;509    }510    std::string builder;511    builder.reserve(s.length());512    size_t pos = 0;513    size_t last_pos = 0;514    while ((pos = s.find(search, last_pos)) != std::string::npos) {515        builder.append(s, last_pos, pos - last_pos);516        builder.append(replace);517        last_pos = pos + search.length();518    }519    builder.append(s, last_pos, std::string::npos);520    s = std::move(builder);521}522 523std::string regex_escape(const std::string & s) {524    static const std::regex special_chars("[.^$|()*+?\\[\\]{}\\\\]");525    return std::regex_replace(s, special_chars, "\\$&");526}527 528std::string string_join(const std::vector<std::string> & values, const std::string & separator) {529    std::ostringstream result;530    for (size_t i = 0; i < values.size(); ++i) {531        if (i > 0) {532            result << separator;533        }534        result << values[i];535    }536    return result.str();537}538 539std::vector<std::string> string_split(const std::string & str, const std::string & delimiter) {540    std::vector<std::string> parts;541    size_t start = 0;542    size_t end = str.find(delimiter);543 544    while (end != std::string::npos) {545        parts.push_back(str.substr(start, end - start));546        start = end + delimiter.length();547        end = str.find(delimiter, start);548    }549 550    parts.push_back(str.substr(start));551 552    return parts;553}554 555std::string string_repeat(const std::string & str, size_t n) {556    if (n == 0) {557        return "";558    }559 560    std::string result;561    result.reserve(str.length() * n);562 563    for (size_t i = 0; i < n; ++i) {564        result += str;565    }566 567    return result;568}569 570std::string string_from(bool value) {571    return value ? "true" : "false";572}573 574std::string string_from(const std::vector<int> & values) {575    std::stringstream buf;576 577    buf << "[ ";578    bool first = true;579    for (auto e : values) {580        if (first) {581            first = false;582        } else {583            buf << ", ";584        }585        buf << std::to_string(e);586    }587    buf << " ]";588 589    return buf.str();590}591 592std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens) {593    std::stringstream buf;594 595    buf << "[ ";596 597    bool first = true;598    for (const auto & token : tokens) {599        if (!first) {600            buf << ", ";601        } else {602            first = false;603        }604 605        auto detokenized = common_token_to_piece(ctx, token);606 607        buf << "'" << detokenized << "'"608            << ":" << std::to_string(token);609    }610 611    buf << " ]";612 613    return buf.str();614}615 616std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {617    std::stringstream buf;618 619    buf << "[ ";620 621    bool first = true;622    for (int i = 0; i < batch.n_tokens; ++i) {623        if (!first) {624            buf << ", ";625        } else {626            first = false;627        }628 629        auto detokenized = common_token_to_piece(ctx, batch.token[i]);630 631        buf << "\n"          << std::to_string(i)632            << ", token '"   << detokenized << "'"633            << ", pos "      << std::to_string(batch.pos[i])634            << ", n_seq_id " << std::to_string(batch.n_seq_id[i])635            << ", seq_id "   << std::to_string(batch.seq_id[i][0])636            << ", logits "   << std::to_string(batch.logits[i]);637    }638 639    buf << " ]";640 641    return buf.str();642}643 644void string_process_escapes(std::string & input) {645    std::size_t input_len = input.length();646    std::size_t output_idx = 0;647 648    for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) {649        if (input[input_idx] == '\\' && input_idx + 1 < input_len) {650            switch (input[++input_idx]) {651                case 'n':  input[output_idx++] = '\n'; break;652                case 'r':  input[output_idx++] = '\r'; break;653                case 't':  input[output_idx++] = '\t'; break;654                case '\'': input[output_idx++] = '\''; break;655                case '\"': input[output_idx++] = '\"'; break;656                case '\\': input[output_idx++] = '\\'; break;657                case 'x':658                    // Handle \x12, etc659                    if (input_idx + 2 < input_len) {660                        const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 };661                        char *err_p = nullptr;662                        const long val = std::strtol(x, &err_p, 16);663                        if (err_p == x + 2) {664                            input_idx += 2;665                            input[output_idx++] = char(val);666                            break;667                        }668                    }669                    // fall through670                default:   input[output_idx++] = '\\';671                           input[output_idx++] = input[input_idx]; break;672            }673        } else {674            input[output_idx++] = input[input_idx];675        }676    }677 678    input.resize(output_idx);679}680 681bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides) {682    const char * sep = strchr(data, '=');683    if (sep == nullptr || sep - data >= 128) {684        COM_ERR("%s: malformed KV override '%s'\n", __func__, data);685        return false;686    }687    llama_model_kv_override kvo;688    std::strncpy(kvo.key, data, sep - data);689    kvo.key[sep - data] = 0;690    sep++;691    if (strncmp(sep, "int:", 4) == 0) {692        sep += 4;693        kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;694        kvo.val_i64 = std::atol(sep);695    } else if (strncmp(sep, "float:", 6) == 0) {696        sep += 6;697        kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT;698        kvo.val_f64 = std::atof(sep);699    } else if (strncmp(sep, "bool:", 5) == 0) {700        sep += 5;701        kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL;702        if (std::strcmp(sep, "true") == 0) {703            kvo.val_bool = true;704        } else if (std::strcmp(sep, "false") == 0) {705            kvo.val_bool = false;706        } else {707            COM_ERR("%s: invalid boolean value for KV override '%s'\n", __func__, data);708            return false;709        }710    } else if (strncmp(sep, "str:", 4) == 0) {711        sep += 4;712        kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;713        if (strlen(sep) > 127) {714            COM_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data);715            return false;716        }717        strncpy(kvo.val_str, sep, 127);718        kvo.val_str[127] = '\0';719    } else {720        COM_ERR("%s: invalid type for KV override '%s'\n", __func__, data);721        return false;722    }723    overrides.emplace_back(std::move(kvo));724    return true;725}726 727static inline bool glob_class_match(const char c, const char * pattern, const char * class_end) {728    const char * class_start = pattern;729    bool negated = false;730 731    if (*class_start == '!') {732        negated = true;733        class_start++;734    }735 736    // If first character after negation is ']' or '-', treat it as literal737    if (*class_start == ']' || *class_start == '-') {738        if (class_start < class_end && *class_start == c) {739            return !negated;740        }741        class_start++;742    }743 744    bool matched = false;745 746    while (class_start < class_end) {747        if (class_start + 2 < class_end && class_start[1] == '-' && class_start[2] != ']') {748            char start_char = *class_start;749            char end_char = class_start[2];750            if (c >= start_char && c <= end_char) {751                matched = true;752                break;753            }754            class_start += 3;755        } else {756            if (*class_start == c) {757                matched = true;758                break;759            }760            class_start++;761        }762    }763 764    return negated ? !matched : matched;765}766 767// simple glob: * matches non-/ chars, ** matches anything including /, [] matches character class768static inline bool glob_match(const char * pattern, const char * str) {769    if (*pattern == '\0') {770        return *str == '\0';771    }772    if (pattern[0] == '*' && pattern[1] == '*') {773        const char * p = pattern + 2;774        if (glob_match(p, str)) return true;775        if (*str != '\0') return glob_match(pattern, str + 1);776        return false;777    }778    if (*pattern == '*') {779        const char * p = pattern + 1;780        for (; *str != '\0' && *str != '/'; str++) {781            if (glob_match(p, str)) return true;782        }783        return glob_match(p, str);784    }785    if (*pattern == '?' && *str != '\0' && *str != '/') {786        return glob_match(pattern + 1, str + 1);787    }788    if (*pattern == '[') {789        const char * class_end = pattern + 1;790        // If first character after '[' is ']' or '-', treat it as literal791        if (*class_end == ']' || *class_end == '-') {792            class_end++;793        }794        while (*class_end != '\0' && *class_end != ']') {795            class_end++;796        }797        if (*class_end == ']') {798            if (*str == '\0') return false;799            bool matched = glob_class_match(*str, pattern + 1, class_end);800            return matched && glob_match(class_end + 1, str + 1);801        } else {802            if (*str == '[') {803                return glob_match(pattern + 1, str + 1);804            }805            return false;806        }807    }808    if (*pattern == *str) {809        return glob_match(pattern + 1, str + 1);810    }811    return false;812}813 814bool glob_match(const std::string & pattern, const std::string & str) {815    return glob_match(pattern.c_str(), str.c_str());816}817 818//819// Filesystem utils820//821 822// Validate if a filename is safe to use823// To validate a full path, split the path by the OS-specific path separator, and validate each part with this function824bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {825    if (!filename.length()) {826        // Empty filename invalid827        return false;828    }829    if (filename.length() > 255) {830        // Limit at common largest possible filename on Linux filesystems831        // to avoid unnecessary further validation832        // (On systems with smaller limits it will be caught by the OS)833        return false;834    }835 836    size_t offset = 0;837    while (offset < filename.size()) {838        utf8_parse_result result = common_parse_utf8_codepoint(filename, offset);839 840        if (result.status != utf8_parse_result::SUCCESS) {841            return false;842        }843        uint32_t c = result.codepoint;844 845        if ((result.bytes_consumed == 2 && c < 0x80) ||846            (result.bytes_consumed == 3 && c < 0x800) ||847            (result.bytes_consumed == 4 && c < 0x10000)) {848            return false;849        }850 851        // Check for forbidden codepoints:852        // - Control characters853        // - Unicode equivalents of illegal characters854        // - UTF-16 surrogate pairs855        // - UTF-8 replacement character856        // - Byte order mark (BOM)857        // - Illegal characters: / \ : * ? " < > |858        if (c <= 0x1F // Control characters (C0)859            || c == 0x7F // Control characters (DEL)860            || (c >= 0x80 && c <= 0x9F) // Control characters (C1)861            || c == 0xFF0E // Fullwidth Full Stop (period equivalent)862            || c == 0x2215 // Division Slash (forward slash equivalent)863            || c == 0x2216 // Set Minus (backslash equivalent)864            || (c >= 0xD800 && c <= 0xDFFF) // UTF-16 surrogate pairs865            || c > 0x10FFFF // Max Unicode limit866            || c == 0xFFFD // Replacement Character (UTF-8)867            || c == 0xFEFF // Byte Order Mark (BOM)868            || c == ':' || c == '*' // Illegal characters869            || c == '?' || c == '"' || c == '<' || c == '>' || c == '|') {870            return false;871        }872        if (!allow_subdirs && (c == '/' || c == '\\')) {873            // Subdirectories not allowed, reject path separators874            return false;875        }876        offset += result.bytes_consumed;877    }878 879    // Reject any leading or trailing ' ', or any trailing '.', these are stripped on Windows and will cause a different filename880    // Unicode and other whitespace is not affected, only 0x20 space881    if (filename.front() == ' ' || filename.back() == ' ' || filename.back() == '.') {882        return false;883    }884 885    // Reject any ".." (currently stricter than necessary, it should be fine to just check for == ".." instead)886    if (filename.find("..") != std::string::npos) {887        return false;888    }889 890    // Reject "."891    if (filename == ".") {892        return false;893    }894 895    return true;896}897 898#include <iostream>899 900 901#ifdef _WIN32902static std::wstring utf8_to_wstring(const std::string & str) {903    if (str.empty()) {904        return std::wstring();905    }906 907    int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);908 909    if (size <= 0) {910        return std::wstring();911    }912 913    std::wstring wstr(size, 0);914    MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);915 916    return wstr;917}918#endif919 920// returns true if successful, false otherwise921bool fs_create_directory_with_parents(const std::string & path) {922#ifdef _WIN32923    std::wstring wpath = utf8_to_wstring(path);924 925    // if the path already exists, check whether it's a directory926    const DWORD attributes = GetFileAttributesW(wpath.c_str());927    if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {928        return true;929    }930 931    size_t pos_slash = 0;932 933    // process path from front to back, procedurally creating directories934    while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {935        const std::wstring subpath = wpath.substr(0, pos_slash);936 937        pos_slash += 1;938 939        // skip the drive letter, in some systems it can return an access denied error940        if (subpath.length() == 2 && subpath[1] == ':') {941            continue;942        }943 944        const bool success = CreateDirectoryW(subpath.c_str(), NULL);945 946        if (!success) {947            const DWORD error = GetLastError();948 949            // if the path already exists, ensure that it's a directory950            if (error == ERROR_ALREADY_EXISTS) {951                const DWORD attributes = GetFileAttributesW(subpath.c_str());952                if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {953                    return false;954                }955            } else {956                return false;957            }958        }959    }960 961    return true;962#else963    // if the path already exists, check whether it's a directory964    struct stat info;965    if (stat(path.c_str(), &info) == 0) {966        return S_ISDIR(info.st_mode);967    }968 969    size_t pos_slash = 1; // skip leading slashes for directory creation970 971    // process path from front to back, procedurally creating directories972    while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {973        const std::string subpath = path.substr(0, pos_slash);974        struct stat info;975 976        // if the path already exists, ensure that it's a directory977        if (stat(subpath.c_str(), &info) == 0) {978            if (!S_ISDIR(info.st_mode)) {979                return false;980            }981        } else {982            // create parent directories983            const int ret = mkdir(subpath.c_str(), 0755);984            if (ret != 0) {985                return false;986            }987        }988 989        pos_slash += 1;990    }991 992    return true;993#endif // _WIN32994}995 996bool fs_is_directory(const std::string & path) {997    std::filesystem::path dir(path);998    return std::filesystem::exists(dir) && std::filesystem::is_directory(dir);999}1000 1001std::string common_get_env(const std::string & name) {1002    const char * value = std::getenv(name.c_str());1003    return value == nullptr ? "" : value;1004}1005 1006void common_set_env(const std::string & name, const std::string & value) {1007#if defined(_WIN32)1008    _putenv_s(name.c_str(), value.c_str());1009#else1010    if (value.empty()) {1011        unsetenv(name.c_str());1012    } else {1013        setenv(name.c_str(), value.c_str(), 1);1014    }1015#endif1016}1017 1018std::string fs_get_cache_directory() {1019    std::string cache_directory = "";1020    auto ensure_trailing_slash = [](std::string p) {1021        // Make sure to add trailing slash1022        if (p.back() != DIRECTORY_SEPARATOR) {1023            p += DIRECTORY_SEPARATOR;1024        }1025        return p;1026    };1027    if (getenv("LLAMA_CACHE")) {1028        cache_directory = std::getenv("LLAMA_CACHE");1029    } else {1030#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \1031        defined(__OpenBSD__) || defined(__NetBSD__)1032        if (std::getenv("XDG_CACHE_HOME")) {1033            cache_directory = std::getenv("XDG_CACHE_HOME");1034        } else if (std::getenv("HOME")) {1035            cache_directory = std::getenv("HOME") + std::string("/.cache/");1036        } else {1037#if defined(__linux__)1038            /* no $HOME is defined, fallback to getpwuid */1039            struct passwd *pw = getpwuid(getuid());1040            if ((!pw) || (!pw->pw_dir)) {1041                throw std::runtime_error("Failed to find $HOME directory");1042            }1043 1044            cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");1045#else /* defined(__linux__) */1046            throw std::runtime_error("Failed to find $HOME directory");1047#endif /* defined(__linux__) */1048        }1049#elif defined(__APPLE__)1050        cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");1051#elif defined(_WIN32)1052        cache_directory = std::getenv("LOCALAPPDATA");1053#elif defined(__EMSCRIPTEN__)1054        GGML_ABORT("not implemented on this platform");1055#else1056#  error Unknown architecture1057#endif1058        cache_directory = ensure_trailing_slash(cache_directory);1059        cache_directory += "llama.cpp";1060    }1061    return ensure_trailing_slash(cache_directory);1062}1063 1064std::string fs_get_cache_file(const std::string & filename) {1065    GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);1066    std::string cache_directory = fs_get_cache_directory();1067    const bool success = fs_create_directory_with_parents(cache_directory);1068    if (!success) {1069        throw std::runtime_error("failed to create cache directory: " + cache_directory);1070    }1071    return cache_directory + filename;1072}1073 1074std::vector<common_file_info> fs_list(const std::string & path, bool include_directories) {1075    std::vector<common_file_info> files;1076    if (path.empty()) return files;1077 1078    std::filesystem::path dir(path);1079    if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) {1080        return files;1081    }1082 1083    for (const auto & entry : std::filesystem::directory_iterator(dir)) {1084        try {1085            // Only include regular files (skip directories)1086            const auto & p = entry.path();1087            if (std::filesystem::is_regular_file(p)) {1088                common_file_info info;1089                info.path   = p.string();1090                info.name   = p.filename().string();1091                info.is_dir = false;1092                try {1093                    info.size = static_cast<size_t>(std::filesystem::file_size(p));1094                } catch (const std::filesystem::filesystem_error &) {1095                    info.size = 0;1096                }1097                files.push_back(std::move(info));1098            } else if (include_directories && std::filesystem::is_directory(p)) {1099                common_file_info info;1100                info.path   = p.string();1101                info.name   = p.filename().string();1102                info.size   = 0; // Directories have no size1103                info.is_dir = true;1104                files.push_back(std::move(info));1105            }1106        } catch (const std::filesystem::filesystem_error &) {1107            // skip entries we cannot inspect1108            continue;1109        }1110    }1111 1112    return files;1113}1114 1115std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) {1116#ifdef _WIN321117    int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);1118    if (!wlen) { return std::ifstream(); }1119    std::vector<wchar_t> wfname(wlen);1120    (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);1121    return std::ifstream(wfname.data(), mode);1122#else1123    return std::ifstream(fname, mode);1124#endif1125}1126 1127//1128// TTY utils1129//1130 1131bool tty_can_use_colors() {1132    // Check NO_COLOR environment variable (https://no-color.org/)1133    if (const char * no_color = std::getenv("NO_COLOR")) {1134        if (no_color[0] != '\0') {1135            return false;1136        }1137    }1138 1139    // Check TERM environment variable1140    if (const char * term = std::getenv("TERM")) {1141        if (std::strcmp(term, "dumb") == 0) {1142            return false;1143        }1144    }1145 1146    // Check if stdout and stderr are connected to a terminal1147    // We check both because log messages can go to either1148    bool stdout_is_tty = isatty(fileno(stdout));1149    bool stderr_is_tty = isatty(fileno(stderr));1150 1151    return stdout_is_tty || stderr_is_tty;1152}1153 1154//1155// Model utils1156//1157 1158// TODO: move to common/sampling1159static void common_init_sampler_from_model(1160    const llama_model * model,1161    common_params_sampling & sparams) {1162 1163    const uint64_t config = sparams.user_sampling_config;1164 1165    auto get_int32 = [&](const char * key, int32_t & dst, uint64_t user_config) {1166        if (config & user_config) {1167            return;1168        }1169 1170        char buf[64] = {0};1171        if (llama_model_meta_val_str(model, key, buf, sizeof(buf)) > 0) {1172            char * end = nullptr;1173            int32_t v = strtol(buf, &end, 10);1174            if (end && end != buf) {1175                dst = v;1176            }1177        }1178    };1179 1180    auto get_float = [&](const char * key, float & dst, uint64_t user_config) {1181        if (config & user_config) {1182            return;1183        }1184 1185        char buf[128] = {0};1186        if (llama_model_meta_val_str(model, key, buf, sizeof(buf)) > 0) {1187            char * end = nullptr;1188            float v = strtof(buf, &end);1189            if (end && end != buf) {1190                dst = v;1191            }1192        }1193    };1194 1195    // Sampling sequence1196    if (!(config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_SAMPLERS)) {1197        char buf[512] = {0};1198        if (llama_model_meta_val_str(model, llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE), buf, sizeof(buf)) > 0) {1199            const std::vector<std::string> sampler_names = string_split<std::string>(std::string(buf), ';');1200            if (!sampler_names.empty()) {

Showing the first 1,200 of 2223 lines. Download the file for the rest.