Brunobkr/llama.cpp_AlgMor24_github

1#define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows2#define _USE_MATH_DEFINES // For M_PI on MSVC3 4#include "ggml-backend.h"5#include "ggml-impl.h"6#include "ggml-threading.h"7#include "ggml-cpu.h"8#include "ggml.h"9 10// FIXME: required here for quantization functions11#include "ggml-quants.h"12 13#ifdef GGML_USE_CPU_HBM14#include <hbwmalloc.h>15#endif16 17#if defined(_MSC_VER) || defined(__MINGW32__)18#include <malloc.h> // using malloc.h with MSC/MINGW19#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)20#include <alloca.h>21#endif22 23#include <assert.h>24#include <errno.h>25#include <time.h>26#include <math.h>27#include <stdlib.h>28#include <string.h>29#include <stdint.h>30#include <inttypes.h>31#include <stdio.h>32#include <float.h>33#include <limits.h>34#include <stdarg.h>35#include <signal.h>36#if defined(__gnu_linux__)37#include <syscall.h>38#endif39 40#if defined(__APPLE__)41#include <unistd.h>42#include <mach/mach.h>43#include <TargetConditionals.h>44#endif45 46#if defined(_WIN32)47#define WIN32_LEAN_AND_MEAN48#ifndef NOMINMAX49    #define NOMINMAX50#endif51#include <windows.h>52#endif53 54#define UNUSED GGML_UNUSED55 56uint64_t ggml_graph_next_uid(void) {57#ifdef _MSC_VER58#if defined(_WIN32)59    static volatile LONG counter = 1;60    return (uint64_t) InterlockedIncrement(&counter) - 1;61#else62    static volatile long long counter = 1;63    return (uint64_t) _InterlockedIncrement64(&counter) - 1;64#endif65#else66    static uint64_t counter = 1;67    return __atomic_fetch_add(&counter, 1, __ATOMIC_RELAXED);68#endif69}70 71// Needed for ggml_fp32_to_bf16_row()72#if defined(__AVX512BF16__)73#if defined(_MSC_VER)74#define m512i(p) p75#else76#include <immintrin.h>77#define m512i(p) (__m512i)(p)78#endif // defined(_MSC_VER)79#endif // defined(__AVX512BF16__)80 81#if defined(__linux__) || \82    defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \83    (defined(__APPLE__) && !TARGET_OS_TV && !TARGET_OS_WATCH)84 85#include <unistd.h>86#include <sys/types.h>87#include <sys/stat.h>88#include <sys/wait.h>89#if defined(__linux__)90#include <sys/prctl.h>91#endif92 93#if defined(__ANDROID__)94#include <unwind.h>95#include <dlfcn.h>96#include <stdio.h>97 98struct backtrace_state {99    void ** current;100    void ** end;101};102 103static _Unwind_Reason_Code unwind_callback(struct _Unwind_Context* context, void* arg) {104    struct backtrace_state * state = (struct backtrace_state *)arg;105    uintptr_t pc = _Unwind_GetIP(context);106    if (pc) {107        if (state->current == state->end) {108            return _URC_END_OF_STACK;109        } else {110            *state->current++ = (void*)pc;111        }112    }113    return _URC_NO_REASON;114}115 116static void ggml_print_backtrace_symbols(void) {117    const int max = 100;118    void* buffer[max];119 120    struct backtrace_state state = {buffer, buffer + max};121    _Unwind_Backtrace(unwind_callback, &state);122 123    int count = state.current - buffer;124 125    for (int idx = 0; idx < count; ++idx) {126        const void * addr = buffer[idx];127        const char * symbol = "";128 129        Dl_info info;130        if (dladdr(addr, &info) && info.dli_sname) {131            symbol = info.dli_sname;132        }133 134        fprintf(stderr, "%d: %p %s\n", idx, addr, symbol);135    }136}137#elif defined(__linux__) && defined(__GLIBC__)138#include <execinfo.h>139static void ggml_print_backtrace_symbols(void) {140    void * trace[100];141    int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));142    backtrace_symbols_fd(trace, nptrs, STDERR_FILENO);143}144#elif defined(__APPLE__)145#include <execinfo.h>146static void ggml_print_backtrace_symbols(void) {147    void * trace[100];148    int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));149    backtrace_symbols_fd(trace, nptrs, STDERR_FILENO);150}151#else152static void ggml_print_backtrace_symbols(void) {153    // platform not supported154}155#endif156 157void ggml_print_backtrace(void) {158    const char * GGML_NO_BACKTRACE = getenv("GGML_NO_BACKTRACE");159    if (GGML_NO_BACKTRACE) {160        return;161    }162#if defined(__APPLE__)163    // On macOS, fork+debugger attachment is problematic due to:164    // 1. libdispatch "poisons" forked child processes165    // 2. lldb has issues attaching to parent from forked child166    // Use simple backtrace() instead to avoid Terminal.app crashes167    const char * GGML_BACKTRACE_LLDB = getenv("GGML_BACKTRACE_LLDB");168    if (!GGML_BACKTRACE_LLDB) {169        fprintf(stderr, "WARNING: Using native backtrace. Set GGML_BACKTRACE_LLDB for more info.\n");170        fprintf(stderr, "WARNING: GGML_BACKTRACE_LLDB may cause native MacOS Terminal.app to crash.\n");171        fprintf(stderr, "See: https://github.com/ggml-org/llama.cpp/pull/17869\n");172        ggml_print_backtrace_symbols();173        return;174    }175#endif176#if defined(__linux__)177    FILE * f = fopen("/proc/self/status", "r");178    size_t size = 0;179    char * line = NULL;180    ssize_t length = 0;181    while ((length = getline(&line, &size, f)) > 0) {182        if (!strncmp(line, "TracerPid:", sizeof("TracerPid:") - 1) &&183            (length != sizeof("TracerPid:\t0\n") - 1 || line[length - 2] != '0')) {184            // Already being debugged, and the breakpoint is the later abort()185            free(line);186            fclose(f);187            return;188        }189    }190    free(line);191    fclose(f);192    int lock[2] = { -1, -1 };193    (void) !pipe(lock); // Don't start gdb until after PR_SET_PTRACER194#endif195    const int parent_pid = getpid();196    const int child_pid = fork();197    if (child_pid < 0) { // error198#if defined(__linux__)199        close(lock[1]);200        close(lock[0]);201#endif202        return;203    } else if (child_pid == 0) { // child204        char attach[32];205        snprintf(attach, sizeof(attach), "attach %d", parent_pid);206#if defined(__linux__)207        close(lock[1]);208        (void) !read(lock[0], lock, 1);209        close(lock[0]);210#endif211        // try gdb212        execlp("gdb", "gdb", "--batch",213            "-ex", "set style enabled on",214            "-ex", attach,215            "-ex", "bt -frame-info source-and-location",216            "-ex", "detach",217            "-ex", "quit",218            (char *) NULL);219        // try lldb220        execlp("lldb", "lldb", "--batch",221            "-o", "bt",222            "-o", "quit",223            "-p", &attach[sizeof("attach ") - 1],224            (char *) NULL);225        // gdb failed, fallback to backtrace_symbols226        ggml_print_backtrace_symbols();227        _Exit(0);228    } else { // parent229#if defined(__linux__)230        prctl(PR_SET_PTRACER, child_pid);231        close(lock[1]);232        close(lock[0]);233#endif234        waitpid(child_pid, NULL, 0);235    }236}237#else238void ggml_print_backtrace(void) {239    // platform not supported240}241#endif242 243static ggml_abort_callback_t g_abort_callback = NULL;244 245// Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)246GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback) {247    ggml_abort_callback_t ret_val = g_abort_callback;248    g_abort_callback = callback;249    return ret_val;250}251 252void ggml_abort(const char * file, int line, const char * fmt, ...) {253    fflush(stdout);254 255    char message[2048];256    int offset = snprintf(message, sizeof(message), "%s:%d: ", file, line);257 258    va_list args;259    va_start(args, fmt);260    vsnprintf(message + offset, sizeof(message) - offset, fmt, args);261    va_end(args);262 263    if (g_abort_callback) {264        g_abort_callback(message);265    } else {266        // default: print error and backtrace to stderr267        fprintf(stderr, "%s\n", message);268        ggml_print_backtrace();269    }270 271    abort();272}273 274// ggml_print_backtrace is registered with std::set_terminate by ggml.cpp275 276//277// logging278//279 280struct ggml_logger_state {281    ggml_log_callback log_callback;282    void * log_callback_user_data;283};284static struct ggml_logger_state g_logger_state = {ggml_log_callback_default, NULL};285 286static void ggml_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {287    if (format == NULL) {288        return;289    }290    va_list args_copy;291    va_copy(args_copy, args);292    char buffer[128];293    int len = vsnprintf(buffer, 128, format, args);294    if (len < 128) {295        g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);296    } else {297        char * buffer2 = (char *) calloc(len + 1, sizeof(char));298        vsnprintf(buffer2, len + 1, format, args_copy);299        buffer2[len] = 0;300        g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);301        free(buffer2);302    }303    va_end(args_copy);304}305 306void ggml_log_internal(enum ggml_log_level level, const char * format, ...) {307    va_list args;308    va_start(args, format);309    ggml_log_internal_v(level, format, args);310    va_end(args);311}312 313void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {314    (void) level;315    (void) user_data;316    fputs(text, stderr);317    fflush(stderr);318}319 320//321// end of logging block322//323 324#ifdef GGML_USE_ACCELERATE325// uncomment to use vDSP for soft max computation326// note: not sure if it is actually faster327//#define GGML_SOFT_MAX_ACCELERATE328#endif329 330 331void * ggml_aligned_malloc(size_t size) {332#if defined(__s390x__)333    const int alignment = 256;334#else335    const int alignment = 64;336#endif337 338#if defined(_MSC_VER) || defined(__MINGW32__)339    return _aligned_malloc(size, alignment);340#else341    if (size == 0) {342        GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");343        return NULL;344    }345    void * aligned_memory = NULL;346  #ifdef GGML_USE_CPU_HBM347    int result = hbw_posix_memalign(&aligned_memory, alignment, size);348  #elif TARGET_OS_OSX349    GGML_UNUSED(alignment);350    kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);351    int result = EFAULT;352    switch (alloc_status) {353        case KERN_SUCCESS:354            result = 0;355            break;356        case KERN_INVALID_ADDRESS:357            result = EINVAL;358            break;359        case KERN_NO_SPACE:360            result = ENOMEM;361            break;362        default:363            result = EFAULT;364            break;365    }366  #else367    int result = posix_memalign(&aligned_memory, alignment, size);368  #endif369    if (result != 0) {370        // Handle allocation failure371        const char *error_desc = "unknown allocation error";372        switch (result) {373            case EINVAL:374                error_desc = "invalid alignment value";375                break;376            case ENOMEM:377                error_desc = "insufficient memory";378                break;379        }380        GGML_LOG_ERROR("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));381        return NULL;382    }383    return aligned_memory;384#endif385}386 387void ggml_aligned_free(void * ptr, size_t size) {388    GGML_UNUSED(size);389#if defined(_MSC_VER) || defined(__MINGW32__)390    _aligned_free(ptr);391#elif GGML_USE_CPU_HBM392    if (ptr != NULL) {393        hbw_free(ptr);394    }395#elif TARGET_OS_OSX396    if (ptr != NULL) {397        vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size);398    }399#else400    free(ptr);401#endif402}403 404 405inline static void * ggml_malloc(size_t size) {406    if (size == 0) {407        GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n");408        return NULL;409    }410    void * result = malloc(size);411    if (result == NULL) {412        GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));413        GGML_ABORT("fatal error");414    }415    return result;416}417 418// calloc419inline static void * ggml_calloc(size_t num, size_t size) {420    if (num == 0 || size == 0) {421        GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n");422        return NULL;423    }424    void * result = calloc(num, size);425    if (result == NULL) {426        GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));427        GGML_ABORT("fatal error");428    }429    return result;430}431 432#define GGML_MALLOC(size)      ggml_malloc(size)433#define GGML_CALLOC(num, size) ggml_calloc(num, size)434 435#define GGML_FREE(ptr) free(ptr)436 437const char * ggml_status_to_string(enum ggml_status status) {438    switch (status) {439        case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";440        case GGML_STATUS_FAILED:       return "GGML status: error (operation failed)";441        case GGML_STATUS_SUCCESS:      return "GGML status: success";442        case GGML_STATUS_ABORTED:      return "GGML status: warning (operation aborted)";443    }444 445    return "GGML status: unknown";446}447 448float ggml_fp16_to_fp32(ggml_fp16_t x) {449#define ggml_fp16_to_fp32 do_not_use__ggml_fp16_to_fp32__in_ggml450    return GGML_FP16_TO_FP32(x);451}452 453ggml_fp16_t ggml_fp32_to_fp16(float x) {454#define ggml_fp32_to_fp16 do_not_use__ggml_fp32_to_fp16__in_ggml455    return GGML_FP32_TO_FP16(x);456}457 458float ggml_bf16_to_fp32(ggml_bf16_t x) {459#define ggml_bf16_to_fp32 do_not_use__ggml_bf16_to_fp32__in_ggml460    return GGML_BF16_TO_FP32(x);  // it just left shifts461}462 463ggml_bf16_t ggml_fp32_to_bf16(float x) {464#define ggml_fp32_to_bf16 do_not_use__ggml_fp32_to_bf16__in_ggml465    return GGML_FP32_TO_BF16(x);466}467 468void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {469    for (int64_t i = 0; i < n; i++) {470        y[i] = GGML_FP16_TO_FP32(x[i]);471    }472}473 474void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {475    int i = 0;476    for (; i < n; ++i) {477        y[i] = GGML_FP32_TO_FP16(x[i]);478    }479}480 481void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {482    int i = 0;483    for (; i < n; ++i) {484        y[i] = GGML_BF16_TO_FP32(x[i]);485    }486}487 488void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {489    for (int i = 0; i < n; i++) {490        y[i] = ggml_compute_fp32_to_bf16(x[i]);491    }492}493 494void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {495  int i = 0;496#if defined(__AVX512BF16__)497  // subnormals are flushed to zero on this platform498  for (; i + 32 <= n; i += 32) {499        _mm512_storeu_si512(500            (__m512i *)(y + i),501            m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),502                                _mm512_loadu_ps(x + i))));503  }504#endif505    for (; i < n; i++) {506        y[i] = GGML_FP32_TO_BF16(x[i]);507    }508}509 510bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {511    return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;512}513 514const char * ggml_version(void) {515    return GGML_VERSION;516}517 518const char * ggml_commit(void) {519    return GGML_COMMIT;520}521 522//523// timing524//525 526#if defined(_MSC_VER) || defined(__MINGW32__)527static int64_t timer_freq, timer_start;528static BOOL CALLBACK ggml_time_init_once(PINIT_ONCE once, PVOID param, PVOID *ctx) {529    UNUSED(once);530    UNUSED(param);531    UNUSED(ctx);532 533    LARGE_INTEGER t;534    QueryPerformanceFrequency(&t);535    timer_freq = t.QuadPart;536 537    // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq538    // and the uptime is high enough.539    // We subtract the program start time to reduce the likelihood of that happening.540    QueryPerformanceCounter(&t);541    timer_start = t.QuadPart;542 543    return TRUE;544}545void ggml_time_init(void) {546    static INIT_ONCE once = INIT_ONCE_STATIC_INIT;547    InitOnceExecuteOnce(&once, ggml_time_init_once, NULL, NULL);548}549int64_t ggml_time_ms(void) {550    LARGE_INTEGER t;551    QueryPerformanceCounter(&t);552    return ((t.QuadPart-timer_start) * 1000) / timer_freq;553}554int64_t ggml_time_us(void) {555    LARGE_INTEGER t;556    QueryPerformanceCounter(&t);557    return ((t.QuadPart-timer_start) * 1000000) / timer_freq;558}559#else560void ggml_time_init(void) {}561int64_t ggml_time_ms(void) {562    struct timespec ts;563    clock_gettime(CLOCK_MONOTONIC, &ts);564    return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;565}566 567int64_t ggml_time_us(void) {568    struct timespec ts;569    clock_gettime(CLOCK_MONOTONIC, &ts);570    return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;571}572#endif573 574int64_t ggml_cycles(void) {575    return clock();576}577 578int64_t ggml_cycles_per_ms(void) {579    return CLOCKS_PER_SEC/1000;580}581 582//583// cross-platform UTF-8 file paths584//585 586#ifdef _WIN32587static wchar_t * ggml_mbstowcs(const char * mbs) {588    int wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, NULL, 0);589    if (!wlen) {590        errno = EINVAL;591        return NULL;592    }593 594    wchar_t * wbuf = GGML_MALLOC(wlen * sizeof(wchar_t));595    wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, wbuf, wlen);596    if (!wlen) {597        GGML_FREE(wbuf);598        errno = EINVAL;599        return NULL;600    }601 602    return wbuf;603}604#endif605 606FILE * ggml_fopen(const char * fname, const char * mode) {607#ifdef _WIN32608    FILE * file = NULL;609 610    // convert fname (UTF-8)611    wchar_t * wfname = ggml_mbstowcs(fname);612    if (wfname) {613        // convert mode (UTF-8)614        wchar_t * wmode = ggml_mbstowcs(mode);615        if (wmode) {616            // open file617            file = _wfopen(wfname, wmode);618            GGML_FREE(wmode);619        }620 621        GGML_FREE(wfname);622    }623 624    return file;625#else626    return fopen(fname, mode);627#endif628 629}630 631static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {632    [GGML_TYPE_I8] = {633        .type_name                = "i8",634        .blck_size                = 1,635        .type_size                = sizeof(int8_t),636        .is_quantized             = false,637    },638    [GGML_TYPE_I16] = {639        .type_name                = "i16",640        .blck_size                = 1,641        .type_size                = sizeof(int16_t),642        .is_quantized             = false,643    },644    [GGML_TYPE_I32] = {645        .type_name                = "i32",646        .blck_size                = 1,647        .type_size                = sizeof(int32_t),648        .is_quantized             = false,649    },650    [GGML_TYPE_I64] = {651        .type_name                = "i64",652        .blck_size                = 1,653        .type_size                = sizeof(int64_t),654        .is_quantized             = false,655    },656    [GGML_TYPE_F64] = {657        .type_name                = "f64",658        .blck_size                = 1,659        .type_size                = sizeof(double),660        .is_quantized             = false,661    },662    [GGML_TYPE_F32] = {663        .type_name                = "f32",664        .blck_size                = 1,665        .type_size                = sizeof(float),666        .is_quantized             = false,667    },668    [GGML_TYPE_F16] = {669        .type_name                = "f16",670        .blck_size                = 1,671        .type_size                = sizeof(ggml_fp16_t),672        .is_quantized             = false,673        .to_float                 = (ggml_to_float_t) ggml_fp16_to_fp32_row,674        .from_float_ref           = (ggml_from_float_t) ggml_fp32_to_fp16_row,675    },676    [GGML_TYPE_Q1_0] = {677        .type_name                = "q1_0",678        .blck_size                = QK1_0,679        .type_size                = sizeof(block_q1_0),680        .is_quantized             = true,681        .to_float                 = (ggml_to_float_t) dequantize_row_q1_0,682        .from_float_ref           = (ggml_from_float_t) quantize_row_q1_0_ref,683    },684    [GGML_TYPE_Q2_0] = {685        .type_name                = "q2_0",686        .blck_size                = QK2_0,687        .type_size                = sizeof(block_q2_0),688        .is_quantized             = true,689        .to_float                 = (ggml_to_float_t) dequantize_row_q2_0,690        .from_float_ref           = (ggml_from_float_t) quantize_row_q2_0_ref,691    },692    [GGML_TYPE_Q4_0] = {693        .type_name                = "q4_0",694        .blck_size                = QK4_0,695        .type_size                = sizeof(block_q4_0),696        .is_quantized             = true,697        .to_float                 = (ggml_to_float_t) dequantize_row_q4_0,698        .from_float_ref           = (ggml_from_float_t) quantize_row_q4_0_ref,699    },700    [GGML_TYPE_Q4_1] = {701        .type_name                = "q4_1",702        .blck_size                = QK4_1,703        .type_size                = sizeof(block_q4_1),704        .is_quantized             = true,705        .to_float                 = (ggml_to_float_t) dequantize_row_q4_1,706        .from_float_ref           = (ggml_from_float_t) quantize_row_q4_1_ref,707    },708    [4] = { // GGML_TYPE_Q4_2709        .type_name                = "DEPRECATED",710        .blck_size                = 0,711        .type_size                = 0,712        .is_quantized             = false,713    },714    [5] = { // GGML_TYPE_Q4_3715        .type_name                = "DEPRECATED",716        .blck_size                = 0,717        .type_size                = 0,718        .is_quantized             = false,719    },720    [GGML_TYPE_Q5_0] = {721        .type_name                = "q5_0",722        .blck_size                = QK5_0,723        .type_size                = sizeof(block_q5_0),724        .is_quantized             = true,725        .to_float                 = (ggml_to_float_t) dequantize_row_q5_0,726        .from_float_ref           = (ggml_from_float_t) quantize_row_q5_0_ref,727    },728    [GGML_TYPE_Q5_1] = {729        .type_name                = "q5_1",730        .blck_size                = QK5_1,731        .type_size                = sizeof(block_q5_1),732        .is_quantized             = true,733        .to_float                 = (ggml_to_float_t) dequantize_row_q5_1,734        .from_float_ref           = (ggml_from_float_t) quantize_row_q5_1_ref,735    },736    [GGML_TYPE_Q8_0] = {737        .type_name                = "q8_0",738        .blck_size                = QK8_0,739        .type_size                = sizeof(block_q8_0),740        .is_quantized             = true,741        .to_float                 = (ggml_to_float_t) dequantize_row_q8_0,742        .from_float_ref           = (ggml_from_float_t) quantize_row_q8_0_ref,743    },744    [GGML_TYPE_Q8_1] = {745        .type_name                = "q8_1",746        .blck_size                = QK8_1,747        .type_size                = sizeof(block_q8_1),748        .is_quantized             = true,749        .from_float_ref           = (ggml_from_float_t) quantize_row_q8_1_ref,750    },751    [GGML_TYPE_MXFP4] = {752        .type_name                = "mxfp4",753        .blck_size                = QK_MXFP4,754        .type_size                = sizeof(block_mxfp4),755        .is_quantized             = true,756        .to_float                 = (ggml_to_float_t) dequantize_row_mxfp4,757        .from_float_ref           = (ggml_from_float_t)quantize_row_mxfp4_ref,758    },759    [GGML_TYPE_NVFP4] = {760        .type_name                = "nvfp4",761        .blck_size                = QK_NVFP4,762        .type_size                = sizeof(block_nvfp4),763        .is_quantized             = true,764        .to_float                 = (ggml_to_float_t) dequantize_row_nvfp4,765        .from_float_ref           = (ggml_from_float_t)quantize_row_nvfp4_ref,766    },767    [GGML_TYPE_Q2_K] = {768        .type_name                = "q2_K",769        .blck_size                = QK_K,770        .type_size                = sizeof(block_q2_K),771        .is_quantized             = true,772        .to_float                 = (ggml_to_float_t) dequantize_row_q2_K,773        .from_float_ref           = (ggml_from_float_t) quantize_row_q2_K_ref,774    },775    [GGML_TYPE_Q3_K] = {776        .type_name                = "q3_K",777        .blck_size                = QK_K,778        .type_size                = sizeof(block_q3_K),779        .is_quantized             = true,780        .to_float                 = (ggml_to_float_t) dequantize_row_q3_K,781        .from_float_ref           = (ggml_from_float_t) quantize_row_q3_K_ref,782    },783    [GGML_TYPE_Q4_K] = {784        .type_name                = "q4_K",785        .blck_size                = QK_K,786        .type_size                = sizeof(block_q4_K),787        .is_quantized             = true,788        .to_float                 = (ggml_to_float_t) dequantize_row_q4_K,789        .from_float_ref           = (ggml_from_float_t) quantize_row_q4_K_ref,790    },791    [GGML_TYPE_Q5_K] = {792        .type_name                = "q5_K",793        .blck_size                = QK_K,794        .type_size                = sizeof(block_q5_K),795        .is_quantized             = true,796        .to_float                 = (ggml_to_float_t) dequantize_row_q5_K,797        .from_float_ref           = (ggml_from_float_t) quantize_row_q5_K_ref,798    },799    [GGML_TYPE_Q6_K] = {800        .type_name                = "q6_K",801        .blck_size                = QK_K,802        .type_size                = sizeof(block_q6_K),803        .is_quantized             = true,804        .to_float                 = (ggml_to_float_t) dequantize_row_q6_K,805        .from_float_ref           = (ggml_from_float_t) quantize_row_q6_K_ref,806    },807    [GGML_TYPE_IQ2_XXS] = {808        .type_name                = "iq2_xxs",809        .blck_size                = QK_K,810        .type_size                = sizeof(block_iq2_xxs),811        .is_quantized             = true,812        .to_float                 = (ggml_to_float_t) dequantize_row_iq2_xxs,813        .from_float_ref           = NULL,814    },815    [GGML_TYPE_IQ2_XS] = {816        .type_name                = "iq2_xs",817        .blck_size                = QK_K,818        .type_size                = sizeof(block_iq2_xs),819        .is_quantized             = true,820        .to_float                 = (ggml_to_float_t) dequantize_row_iq2_xs,821        .from_float_ref           = NULL,822    },823    [GGML_TYPE_IQ3_XXS] = {824        .type_name                = "iq3_xxs",825        .blck_size                = QK_K,826        .type_size                = sizeof(block_iq3_xxs),827        .is_quantized             = true,828        .to_float                 = (ggml_to_float_t) dequantize_row_iq3_xxs,829        .from_float_ref           = (ggml_from_float_t)quantize_row_iq3_xxs_ref,830    },831    [GGML_TYPE_IQ3_S] = {832        .type_name                = "iq3_s",833        .blck_size                = QK_K,834        .type_size                = sizeof(block_iq3_s),835        .is_quantized             = true,836        .to_float                 = (ggml_to_float_t) dequantize_row_iq3_s,837        .from_float_ref           = (ggml_from_float_t)quantize_row_iq3_s_ref,838    },839    [GGML_TYPE_IQ2_S] = {840        .type_name                = "iq2_s",841        .blck_size                = QK_K,842        .type_size                = sizeof(block_iq2_s),843        .is_quantized             = true,844        .to_float                 = (ggml_to_float_t) dequantize_row_iq2_s,845        .from_float_ref           = (ggml_from_float_t)quantize_row_iq2_s_ref,846    },847    [GGML_TYPE_IQ1_S] = {848        .type_name                = "iq1_s",849        .blck_size                = QK_K,850        .type_size                = sizeof(block_iq1_s),851        .is_quantized             = true,852        .to_float                 = (ggml_to_float_t) dequantize_row_iq1_s,853        .from_float_ref           = NULL,854    },855    [GGML_TYPE_IQ1_M] = {856        .type_name                = "iq1_m",857        .blck_size                = QK_K,858        .type_size                = sizeof(block_iq1_m),859        .is_quantized             = true,860        .to_float                 = (ggml_to_float_t) dequantize_row_iq1_m,861        .from_float_ref           = NULL,862    },863    [GGML_TYPE_IQ4_NL] = {864        .type_name                = "iq4_nl",865        .blck_size                = QK4_NL,866        .type_size                = sizeof(block_iq4_nl),867        .is_quantized             = true,868        .to_float                 = (ggml_to_float_t) dequantize_row_iq4_nl,869        .from_float_ref           = (ggml_from_float_t)quantize_row_iq4_nl_ref,870    },871    [GGML_TYPE_IQ4_XS] = {872        .type_name                = "iq4_xs",873        .blck_size                = QK_K,874        .type_size                = sizeof(block_iq4_xs),875        .is_quantized             = true,876        .to_float                 = (ggml_to_float_t) dequantize_row_iq4_xs,877        .from_float_ref           = (ggml_from_float_t)quantize_row_iq4_xs_ref,878    },879    [GGML_TYPE_Q8_K] = {880        .type_name                = "q8_K",881        .blck_size                = QK_K,882        .type_size                = sizeof(block_q8_K),883        .is_quantized             = true,884    },885    [GGML_TYPE_BF16] = {886        .type_name                = "bf16",887        .blck_size                = 1,888        .type_size                = sizeof(ggml_bf16_t),889        .is_quantized             = false,890        .to_float                 = (ggml_to_float_t) ggml_bf16_to_fp32_row,891        .from_float_ref           = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,892    },893    [31] = { // GGML_TYPE_Q4_0_4_4894        .type_name                = "TYPE_Q4_0_4_4 REMOVED, use Q4_0 with runtime repacking",895        .blck_size                = 0,896        .type_size                = 0,897        .is_quantized             = false,898    },899    [32] = { // GGML_TYPE_Q4_0_4_8900        .type_name                = "TYPE_Q4_0_4_8 REMOVED, use Q4_0 with runtime repacking",901        .blck_size                = 0,902        .type_size                = 0,903        .is_quantized             = false,904    },905    [33] = { // GGML_TYPE_Q4_0_8_8906        .type_name                = "TYPE_Q4_0_8_8 REMOVED, use Q4_0 with runtime repacking",907        .blck_size                = 0,908        .type_size                = 0,909        .is_quantized             = false,910    },911    [GGML_TYPE_TQ1_0] = {912        .type_name                = "tq1_0",913        .blck_size                = QK_K,914        .type_size                = sizeof(block_tq1_0),915        .is_quantized             = true,916        .to_float                 = (ggml_to_float_t) dequantize_row_tq1_0,917        .from_float_ref           = (ggml_from_float_t) quantize_row_tq1_0_ref,918    },919    [GGML_TYPE_TQ2_0] = {920        .type_name                = "tq2_0",921        .blck_size                = QK_K,922        .type_size                = sizeof(block_tq2_0),923        .is_quantized             = true,924        .to_float                 = (ggml_to_float_t) dequantize_row_tq2_0,925        .from_float_ref           = (ggml_from_float_t) quantize_row_tq2_0_ref,926    },927    [36] = { // GGML_TYPE_IQ4_NL_4_4928        .type_name                = "TYPE_IQ4_NL_4_4 REMOVED, use IQ4_NL with runtime repacking",929        .blck_size                = 0,930        .type_size                = 0,931        .is_quantized             = false,932    },933    [37] = { // GGML_TYPE_IQ4_NL_4_8934        .type_name                = "TYPE_IQ4_NL_4_8 REMOVED, use IQ4_NL with runtime repacking",935        .blck_size                = 0,936        .type_size                = 0,937        .is_quantized             = false,938    },939    [38] = { // GGML_TYPE_IQ4_NL_8_8940        .type_name                = "TYPE_IQ4_NL_8_8 REMOVED, use IQ4_NL with runtime repacking",941        .blck_size                = 0,942        .type_size                = 0,943        .is_quantized             = false,944    },945};946 947const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type) {948    assert(type >= 0);949    assert(type < GGML_TYPE_COUNT);950    return &type_traits[type];951}952 953//954// ggml object955//956 957struct ggml_object {958    size_t offs;959    size_t size;960 961    struct ggml_object * next;962 963    enum ggml_object_type type;964 965    char padding[4];966};967 968static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);969 970//971// ggml context972//973 974struct ggml_context {975    size_t mem_size;976    void * mem_buffer;977    bool   mem_buffer_owned;978    bool   no_alloc;979 980    int    n_objects;981 982    struct ggml_object * objects_begin;983    struct ggml_object * objects_end;984};985 986//987// data types988//989 990static const char * GGML_OP_NAME[GGML_OP_COUNT] = {991    "NONE",992 993    "DUP",994    "ADD",995    "ADD_ID",996    "ADD1",997    "ACC",998    "SUB",999    "MUL",1000    "DIV",1001    "SQR",1002    "SQRT",1003    "LOG",1004    "SIN",1005    "COS",1006    "SUM",1007    "SUM_ROWS",1008    "CUMSUM",1009    "MEAN",1010    "ARGMAX",1011    "COUNT_EQUAL",1012    "REPEAT",1013    "REPEAT_BACK",1014    "CONCAT",1015    "SILU_BACK",1016    "NORM",1017    "RMS_NORM",1018    "RMS_NORM_BACK",1019    "GROUP_NORM",1020    "L2_NORM",1021 1022    "MUL_MAT",1023    "MUL_MAT_ID",1024    "OUT_PROD",1025 1026    "SCALE",1027    "SET",1028    "CPY",1029    "CONT",1030    "RESHAPE",1031    "VIEW",1032    "PERMUTE",1033    "TRANSPOSE",1034    "GET_ROWS",1035    "GET_ROWS_BACK",1036    "SET_ROWS",1037    "DIAG",1038    "DIAG_MASK_INF",1039    "DIAG_MASK_ZERO",1040    "SOFT_MAX",1041    "SOFT_MAX_BACK",1042    "ROPE",1043    "ROPE_BACK",1044    "CLAMP",1045    "CONV_TRANSPOSE_1D",1046    "IM2COL",1047    "IM2COL_BACK",1048    "IM2COL_3D",1049    "COL2IM_1D",1050    "CONV_2D",1051    "CONV_3D",1052    "CONV_2D_DW",1053    "CONV_TRANSPOSE_2D",1054    "POOL_1D",1055    "POOL_2D",1056    "POOL_2D_BACK",1057    "UPSCALE",1058    "PAD",1059    "PAD_REFLECT_1D",1060    "ROLL",1061    "ARANGE",1062    "TIMESTEP_EMBEDDING",1063    "ARGSORT",1064    "TOP_K",1065    "LEAKY_RELU",1066    "TRI",1067    "FILL",1068 1069    "FLASH_ATTN_EXT",1070    "FLASH_ATTN_BACK",1071    "SSM_CONV",1072    "SSM_SCAN",1073    "WIN_PART",1074    "WIN_UNPART",1075    "GET_REL_POS",1076    "ADD_REL_POS",1077    "RWKV_WKV6",1078    "GATED_LINEAR_ATTN",1079    "RWKV_WKV7",1080    "SOLVE_TRI",1081    "GATED_DELTA_NET",1082    "LIGHTNING_INDEXER",1083    "DSV4_HC_COMB",1084    "DSV4_HC_PRE",1085    "DSV4_HC_POST",1086 1087    "UNARY",1088 1089    "MAP_CUSTOM1",1090    "MAP_CUSTOM2",1091    "MAP_CUSTOM3",1092 1093    "CUSTOM",1094 1095    "CROSS_ENTROPY_LOSS",1096    "CROSS_ENTROPY_LOSS_BACK",1097    "OPT_STEP_ADAMW",1098    "OPT_STEP_SGD",1099 1100    "GLU",1101};1102 1103static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101");1104 1105static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {1106    "none",1107 1108    "x",1109    "x+y",1110    "x[i]+y",1111    "x+y",1112    "view(x,nb,offset)+=y->x",1113    "x-y",1114    "x*y",1115    "x/y",1116    "x^2",1117    "√x",1118    "log(x)",1119    "sin(x)",1120    "cos(x)",1121    "Σx",1122    "Σx_k",1123    "cumsum(x)",1124    "Σx/n",1125    "argmax(x)",1126    "count_equal(x)",1127    "repeat(x)",1128    "repeat_back(x)",1129    "concat(x, y)",1130    "silu_back(x)",1131    "norm(x)",1132    "rms_norm(x)",1133    "rms_norm_back(x)",1134    "group_norm(x)",1135    "l2_norm(x)",1136 1137    "X*Y",1138    "X[i]*Y",1139    "X*Y",1140 1141    "x*v",1142    "y-\\>view(x)",1143    "x-\\>y",1144    "cont(x)",1145    "reshape(x)",1146    "view(x)",1147    "permute(x)",1148    "transpose(x)",1149    "get_rows(x)",1150    "get_rows_back(x)",1151    "set_rows(x)",1152    "diag(x)",1153    "diag_mask_inf(x)",1154    "diag_mask_zero(x)",1155    "soft_max(x)",1156    "soft_max_back(x)",1157    "rope(x)",1158    "rope_back(x)",1159    "clamp(x)",1160    "conv_transpose_1d(x)",1161    "im2col(x)",1162    "im2col_back(x)",1163    "im2col_3d(x)",1164    "col2im_1d(x)",1165    "conv_2d(x)",1166    "conv_3d(x)",1167    "conv_2d_dw(x)",1168    "conv_transpose_2d(x)",1169    "pool_1d(x)",1170    "pool_2d(x)",1171    "pool_2d_back(x)",1172    "upscale(x)",1173    "pad(x)",1174    "pad_reflect_1d(x)",1175    "roll(x)",1176    "arange(start, stop, step)",1177    "timestep_embedding(timesteps, dim, max_period)",1178    "argsort(x)",1179    "top_k(x)",1180    "leaky_relu(x)",1181    "tri(x)",1182    "fill(x, c)",1183 1184    "flash_attn_ext(x)",1185    "flash_attn_back(x)",1186    "ssm_conv(x)",1187    "ssm_scan(x)",1188    "win_part(x)",1189    "win_unpart(x)",1190    "get_rel_pos(x)",1191    "add_rel_pos(x)",1192    "rwkv_wkv6(k, v, r, tf, td, s)",1193    "gated_linear_attn(k, v, q, gate, s)",1194    "rwkv_wkv7(r, w, k, v, a, b, s)",1195    "A X = B, A triangular, solve X",1196    "gated_delta_net(q, k, v, g, beta, s)",1197    "lightning_indexer(q, k, weights, mask)",1198    "dsv4_hc_comb(mixes, scale, base)",1199    "dsv4_hc_pre(x, weights)",1200    "dsv4_hc_post(x, residual, post, comb)",

Showing the first 1,200 of 8030 lines. Download the file for the rest.