KBaba7/llama.cpp
0
1#define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows2#define _USE_MATH_DEFINES // For M_PI on MSVC3 4#include "ggml-backend.h"5#include "ggml-impl.h"6#include "ggml-threading.h"7#include "ggml.h"8 9// FIXME: required here for quantization functions10#include "ggml-quants.h"11 12#ifdef GGML_USE_CPU_HBM13#include <hbwmalloc.h>14#endif15 16#if defined(_MSC_VER) || defined(__MINGW32__)17#include <malloc.h> // using malloc.h with MSC/MINGW18#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)19#include <alloca.h>20#endif21 22#include <assert.h>23#include <errno.h>24#include <time.h>25#include <math.h>26#include <stdlib.h>27#include <string.h>28#include <stdint.h>29#include <inttypes.h>30#include <stdio.h>31#include <float.h>32#include <limits.h>33#include <stdarg.h>34#include <signal.h>35#if defined(__gnu_linux__)36#include <syscall.h>37#endif38 39#if defined(__APPLE__)40#include <unistd.h>41#include <mach/mach.h>42#include <TargetConditionals.h>43#endif44 45#if defined(_WIN32)46#define WIN32_LEAN_AND_MEAN47#ifndef NOMINMAX48 #define NOMINMAX49#endif50#include <windows.h>51#endif52 53#define UNUSED GGML_UNUSED54 55#if defined(_MSC_VER)56#define m512bh(p) p57#define m512i(p) p58#else59#define m512bh(p) (__m512bh)(p)60#define m512i(p) (__m512i)(p)61#endif62 63// precomputed f32 table for f16 (256 KB) (ggml-impl.h)64float ggml_table_f32_f16[1 << 16];65 66#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)) && \67 (!defined(TARGET_OS_TV) && !defined(TARGET_OS_WATCH))68#include <unistd.h>69#include <sys/types.h>70#include <sys/stat.h>71#include <sys/wait.h>72 73#if defined(__ANDROID__)74#include <unwind.h>75#include <dlfcn.h>76#include <stdio.h>77 78struct backtrace_state {79 void ** current;80 void ** end;81};82 83static _Unwind_Reason_Code unwind_callback(struct _Unwind_Context* context, void* arg) {84 struct backtrace_state * state = (struct backtrace_state *)arg;85 uintptr_t pc = _Unwind_GetIP(context);86 if (pc) {87 if (state->current == state->end) {88 return _URC_END_OF_STACK;89 } else {90 *state->current++ = (void*)pc;91 }92 }93 return _URC_NO_REASON;94}95 96static void ggml_print_backtrace_symbols(void) {97 const int max = 100;98 void* buffer[max];99 100 struct backtrace_state state = {buffer, buffer + max};101 _Unwind_Backtrace(unwind_callback, &state);102 103 int count = state.current - buffer;104 105 for (int idx = 0; idx < count; ++idx) {106 const void * addr = buffer[idx];107 const char * symbol = "";108 109 Dl_info info;110 if (dladdr(addr, &info) && info.dli_sname) {111 symbol = info.dli_sname;112 }113 114 fprintf(stderr, "%d: %p %s\n", idx, addr, symbol);115 }116}117#elif defined(__linux__) && defined(__GLIBC__)118#include <execinfo.h>119static void ggml_print_backtrace_symbols(void) {120 void * trace[100];121 int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));122 backtrace_symbols_fd(trace, nptrs, STDERR_FILENO);123}124#else125static void ggml_print_backtrace_symbols(void) {126 // platform not supported127}128#endif129 130static void ggml_print_backtrace(void) {131 const char * GGML_NO_BACKTRACE = getenv("GGML_NO_BACKTRACE");132 if (GGML_NO_BACKTRACE) {133 return;134 }135 char attach[32];136 snprintf(attach, sizeof(attach), "attach %d", getpid());137 int pid = fork();138 if (pid == 0) {139 // try gdb140 execlp("gdb", "gdb", "--batch",141 "-ex", "set style enabled on",142 "-ex", attach,143 "-ex", "bt -frame-info source-and-location",144 "-ex", "detach",145 "-ex", "quit",146 (char *) NULL);147 // try lldb148 execlp("lldb", "lldb", "--batch",149 "-o", "bt",150 "-o", "quit",151 "-p", attach,152 (char *) NULL);153 exit(EXIT_FAILURE);154 } else {155 int wstatus;156 waitpid(pid, &wstatus, 0);157 if (WIFEXITED(wstatus)) {158 if (WEXITSTATUS(wstatus) == EXIT_FAILURE) {159 // gdb failed, fallback to backtrace_symbols160 ggml_print_backtrace_symbols();161 }162 }163 }164}165#else166static void ggml_print_backtrace(void) {167 // platform not supported168}169#endif170 171void ggml_abort(const char * file, int line, const char * fmt, ...) {172 fflush(stdout);173 174 fprintf(stderr, "%s:%d: ", file, line);175 176 va_list args;177 va_start(args, fmt);178 vfprintf(stderr, fmt, args);179 va_end(args);180 181 fprintf(stderr, "\n");182 183 ggml_print_backtrace();184 abort();185}186 187//188// logging189//190 191struct ggml_logger_state {192 ggml_log_callback log_callback;193 void * log_callback_user_data;194};195static struct ggml_logger_state g_logger_state = {ggml_log_callback_default, NULL};196 197static void ggml_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {198 if (format == NULL) {199 return;200 }201 va_list args_copy;202 va_copy(args_copy, args);203 char buffer[128];204 int len = vsnprintf(buffer, 128, format, args);205 if (len < 128) {206 g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);207 } else {208 char * buffer2 = (char *) calloc(len + 1, sizeof(char));209 vsnprintf(buffer2, len + 1, format, args_copy);210 buffer2[len] = 0;211 g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);212 free(buffer2);213 }214 va_end(args_copy);215}216 217void ggml_log_internal(enum ggml_log_level level, const char * format, ...) {218 va_list args;219 va_start(args, format);220 ggml_log_internal_v(level, format, args);221 va_end(args);222}223 224void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {225 (void) level;226 (void) user_data;227 fputs(text, stderr);228 fflush(stderr);229}230 231//232// end of logging block233//234 235#ifdef GGML_USE_ACCELERATE236// uncomment to use vDSP for soft max computation237// note: not sure if it is actually faster238//#define GGML_SOFT_MAX_ACCELERATE239#endif240 241 242void * ggml_aligned_malloc(size_t size) {243 const int alignment = 64;244 245#if defined(_MSC_VER) || defined(__MINGW32__)246 return _aligned_malloc(size, alignment);247#else248 if (size == 0) {249 GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");250 return NULL;251 }252 void * aligned_memory = NULL;253 #ifdef GGML_USE_CPU_HBM254 int result = hbw_posix_memalign(&aligned_memory, alignment, size);255 #elif TARGET_OS_OSX256 GGML_UNUSED(alignment);257 kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);258 int result = EFAULT;259 switch (alloc_status) {260 case KERN_SUCCESS:261 result = 0;262 break;263 case KERN_INVALID_ADDRESS:264 result = EINVAL;265 break;266 case KERN_NO_SPACE:267 result = ENOMEM;268 break;269 default:270 result = EFAULT;271 break;272 }273 #else274 int result = posix_memalign(&aligned_memory, alignment, size);275 #endif276 if (result != 0) {277 // Handle allocation failure278 const char *error_desc = "unknown allocation error";279 switch (result) {280 case EINVAL:281 error_desc = "invalid alignment value";282 break;283 case ENOMEM:284 error_desc = "insufficient memory";285 break;286 }287 GGML_LOG_ERROR("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));288 return NULL;289 }290 return aligned_memory;291#endif292}293 294void ggml_aligned_free(void * ptr, size_t size) {295 GGML_UNUSED(size);296#if defined(_MSC_VER) || defined(__MINGW32__)297 _aligned_free(ptr);298#elif GGML_USE_CPU_HBM299 if (ptr != NULL) {300 hbw_free(ptr);301 }302#elif TARGET_OS_OSX303 if (ptr != NULL) {304 vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size);305 }306#else307 free(ptr);308#endif309}310 311 312inline static void * ggml_malloc(size_t size) {313 if (size == 0) {314 GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n");315 return NULL;316 }317 void * result = malloc(size);318 if (result == NULL) {319 GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));320 GGML_ABORT("fatal error");321 }322 return result;323}324 325// calloc326inline static void * ggml_calloc(size_t num, size_t size) {327 if (num == 0 || size == 0) {328 GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n");329 return NULL;330 }331 void * result = calloc(num, size);332 if (result == NULL) {333 GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));334 GGML_ABORT("fatal error");335 }336 return result;337}338 339#define GGML_MALLOC(size) ggml_malloc(size)340#define GGML_CALLOC(num, size) ggml_calloc(num, size)341 342#define GGML_FREE(ptr) free(ptr)343 344const char * ggml_status_to_string(enum ggml_status status) {345 switch (status) {346 case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";347 case GGML_STATUS_FAILED: return "GGML status: error (operation failed)";348 case GGML_STATUS_SUCCESS: return "GGML status: success";349 case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)";350 }351 352 return "GGML status: unknown";353}354 355float ggml_fp16_to_fp32(ggml_fp16_t x) {356#define ggml_fp16_to_fp32 do_not_use__ggml_fp16_to_fp32__in_ggml357 return GGML_FP16_TO_FP32(x);358}359 360ggml_fp16_t ggml_fp32_to_fp16(float x) {361#define ggml_fp32_to_fp16 do_not_use__ggml_fp32_to_fp16__in_ggml362 return GGML_FP32_TO_FP16(x);363}364 365float ggml_bf16_to_fp32(ggml_bf16_t x) {366#define ggml_bf16_to_fp32 do_not_use__ggml_bf16_to_fp32__in_ggml367 return GGML_BF16_TO_FP32(x); // it just left shifts368}369 370ggml_bf16_t ggml_fp32_to_bf16(float x) {371#define ggml_fp32_to_bf16 do_not_use__ggml_fp32_to_bf16__in_ggml372 return GGML_FP32_TO_BF16(x);373}374 375void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {376 for (int64_t i = 0; i < n; i++) {377 y[i] = GGML_FP16_TO_FP32(x[i]);378 }379}380 381// FIXME: these functions must detect the instruction set at runtime, since they are part of the core ggml library382// currently, the ggml_cpu_has_* functions are entirely compile-time383void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {384 int64_t i = 0;385#if defined(__F16C__)386 //if (ggml_cpu_has_f16c()) {387 for (; i + 7 < n; i += 8) {388 __m256 x_vec = _mm256_loadu_ps(x + i);389 __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);390 _mm_storeu_si128((__m128i *)(y + i), y_vec);391 }392 for(; i + 3 < n; i += 4) {393 __m128 x_vec = _mm_loadu_ps(x + i);394 __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);395 _mm_storel_epi64((__m128i *)(y + i), y_vec);396 }397 //}398#endif399 for (; i < n; i++) {400 y[i] = GGML_FP32_TO_FP16(x[i]);401 }402}403 404void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {405 int64_t i = 0;406#if defined(__AVX512F__)407 //if (ggml_cpu_has_avx512()) {408 for (; i + 16 <= n; i += 16) {409 _mm512_storeu_ps(y + i,410 _mm512_castsi512_ps(411 _mm512_slli_epi32(412 _mm512_cvtepu16_epi32(413 _mm256_loadu_si256(414 (const __m256i *)(x + i))),415 16)));416 }417 //}418#endif419#if defined(__AVX2__)420 //if (ggml_cpu_has_avx2()) {421 for (; i + 8 <= n; i += 8) {422 _mm256_storeu_ps(y + i,423 _mm256_castsi256_ps(424 _mm256_slli_epi32(425 _mm256_cvtepu16_epi32(426 _mm_loadu_si128(427 (const __m128i *)(x + i))),428 16)));429 }430 //}431#endif432 for (; i < n; i++) {433 y[i] = GGML_BF16_TO_FP32(x[i]);434 }435}436 437void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {438 for (int i = 0; i < n; i++) {439 y[i] = ggml_compute_fp32_to_bf16(x[i]);440 }441}442 443void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {444 int i = 0;445#if defined(__AVX512BF16__)446 // subnormals are flushed to zero on this platform447 for (; i + 32 <= n; i += 32) {448 _mm512_storeu_si512(449 (__m512i *)(y + i),450 m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),451 _mm512_loadu_ps(x + i))));452 }453#endif454 for (; i < n; i++) {455 y[i] = GGML_FP32_TO_BF16(x[i]);456 }457}458 459bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {460 return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;461}462 463//464// timing465//466 467#if defined(_MSC_VER) || defined(__MINGW32__)468static int64_t timer_freq, timer_start;469void ggml_time_init(void) {470 LARGE_INTEGER t;471 QueryPerformanceFrequency(&t);472 timer_freq = t.QuadPart;473 474 // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq475 // and the uptime is high enough.476 // We subtract the program start time to reduce the likelihood of that happening.477 QueryPerformanceCounter(&t);478 timer_start = t.QuadPart;479}480int64_t ggml_time_ms(void) {481 LARGE_INTEGER t;482 QueryPerformanceCounter(&t);483 return ((t.QuadPart-timer_start) * 1000) / timer_freq;484}485int64_t ggml_time_us(void) {486 LARGE_INTEGER t;487 QueryPerformanceCounter(&t);488 return ((t.QuadPart-timer_start) * 1000000) / timer_freq;489}490#else491void ggml_time_init(void) {}492int64_t ggml_time_ms(void) {493 struct timespec ts;494 clock_gettime(CLOCK_MONOTONIC, &ts);495 return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;496}497 498int64_t ggml_time_us(void) {499 struct timespec ts;500 clock_gettime(CLOCK_MONOTONIC, &ts);501 return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;502}503#endif504 505int64_t ggml_cycles(void) {506 return clock();507}508 509int64_t ggml_cycles_per_ms(void) {510 return CLOCKS_PER_SEC/1000;511}512 513//514// cross-platform UTF-8 file paths515//516 517#ifdef _WIN32518static wchar_t * ggml_mbstowcs(const char * mbs) {519 int wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, NULL, 0);520 if (!wlen) {521 errno = EINVAL;522 return NULL;523 }524 525 wchar_t * wbuf = GGML_MALLOC(wlen * sizeof(wchar_t));526 wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, wbuf, wlen);527 if (!wlen) {528 GGML_FREE(wbuf);529 errno = EINVAL;530 return NULL;531 }532 533 return wbuf;534}535#endif536 537FILE * ggml_fopen(const char * fname, const char * mode) {538#ifdef _WIN32539 FILE * file = NULL;540 541 // convert fname (UTF-8)542 wchar_t * wfname = ggml_mbstowcs(fname);543 if (wfname) {544 // convert mode (ANSI)545 wchar_t * wmode = GGML_MALLOC((strlen(mode) + 1) * sizeof(wchar_t));546 wchar_t * wmode_p = wmode;547 do {548 *wmode_p++ = (wchar_t)*mode;549 } while (*mode++);550 551 // open file552 file = _wfopen(wfname, wmode);553 554 GGML_FREE(wfname);555 GGML_FREE(wmode);556 }557 558 return file;559#else560 return fopen(fname, mode);561#endif562 563}564static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);565static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);566static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);567 568static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {569 [GGML_TYPE_I8] = {570 .type_name = "i8",571 .blck_size = 1,572 .type_size = sizeof(int8_t),573 .is_quantized = false,574 },575 [GGML_TYPE_I16] = {576 .type_name = "i16",577 .blck_size = 1,578 .type_size = sizeof(int16_t),579 .is_quantized = false,580 },581 [GGML_TYPE_I32] = {582 .type_name = "i32",583 .blck_size = 1,584 .type_size = sizeof(int32_t),585 .is_quantized = false,586 },587 [GGML_TYPE_I64] = {588 .type_name = "i64",589 .blck_size = 1,590 .type_size = sizeof(int64_t),591 .is_quantized = false,592 },593 [GGML_TYPE_F64] = {594 .type_name = "f64",595 .blck_size = 1,596 .type_size = sizeof(double),597 .is_quantized = false,598 },599 [GGML_TYPE_F32] = {600 .type_name = "f32",601 .blck_size = 1,602 .type_size = sizeof(float),603 .is_quantized = false,604 },605 [GGML_TYPE_F16] = {606 .type_name = "f16",607 .blck_size = 1,608 .type_size = sizeof(ggml_fp16_t),609 .is_quantized = false,610 .to_float = (ggml_to_float_t) ggml_fp16_to_fp32_row,611 .from_float_ref = (ggml_from_float_t) ggml_fp32_to_fp16_row,612 },613 [GGML_TYPE_Q4_0] = {614 .type_name = "q4_0",615 .blck_size = QK4_0,616 .type_size = sizeof(block_q4_0),617 .is_quantized = true,618 .to_float = (ggml_to_float_t) dequantize_row_q4_0,619 .from_float_ref = (ggml_from_float_t) quantize_row_q4_0_ref,620 },621 [GGML_TYPE_Q4_1] = {622 .type_name = "q4_1",623 .blck_size = QK4_1,624 .type_size = sizeof(block_q4_1),625 .is_quantized = true,626 .to_float = (ggml_to_float_t) dequantize_row_q4_1,627 .from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref,628 },629 [4] = { // GGML_TYPE_Q4_2630 .type_name = "DEPRECATED",631 .blck_size = 0,632 .type_size = 0,633 .is_quantized = false,634 },635 [5] = { // GGML_TYPE_Q4_3636 .type_name = "DEPRECATED",637 .blck_size = 0,638 .type_size = 0,639 .is_quantized = false,640 },641 [GGML_TYPE_Q5_0] = {642 .type_name = "q5_0",643 .blck_size = QK5_0,644 .type_size = sizeof(block_q5_0),645 .is_quantized = true,646 .to_float = (ggml_to_float_t) dequantize_row_q5_0,647 .from_float_ref = (ggml_from_float_t) quantize_row_q5_0_ref,648 },649 [GGML_TYPE_Q5_1] = {650 .type_name = "q5_1",651 .blck_size = QK5_1,652 .type_size = sizeof(block_q5_1),653 .is_quantized = true,654 .to_float = (ggml_to_float_t) dequantize_row_q5_1,655 .from_float_ref = (ggml_from_float_t) quantize_row_q5_1_ref,656 },657 [GGML_TYPE_Q8_0] = {658 .type_name = "q8_0",659 .blck_size = QK8_0,660 .type_size = sizeof(block_q8_0),661 .is_quantized = true,662 .to_float = (ggml_to_float_t) dequantize_row_q8_0,663 .from_float_ref = (ggml_from_float_t) quantize_row_q8_0_ref,664 },665 [GGML_TYPE_Q8_1] = {666 .type_name = "q8_1",667 .blck_size = QK8_1,668 .type_size = sizeof(block_q8_1),669 .is_quantized = true,670 .from_float_ref = (ggml_from_float_t) quantize_row_q8_1_ref,671 },672 [GGML_TYPE_Q2_K] = {673 .type_name = "q2_K",674 .blck_size = QK_K,675 .type_size = sizeof(block_q2_K),676 .is_quantized = true,677 .to_float = (ggml_to_float_t) dequantize_row_q2_K,678 .from_float_ref = (ggml_from_float_t) quantize_row_q2_K_ref,679 },680 [GGML_TYPE_Q3_K] = {681 .type_name = "q3_K",682 .blck_size = QK_K,683 .type_size = sizeof(block_q3_K),684 .is_quantized = true,685 .to_float = (ggml_to_float_t) dequantize_row_q3_K,686 .from_float_ref = (ggml_from_float_t) quantize_row_q3_K_ref,687 },688 [GGML_TYPE_Q4_K] = {689 .type_name = "q4_K",690 .blck_size = QK_K,691 .type_size = sizeof(block_q4_K),692 .is_quantized = true,693 .to_float = (ggml_to_float_t) dequantize_row_q4_K,694 .from_float_ref = (ggml_from_float_t) quantize_row_q4_K_ref,695 },696 [GGML_TYPE_Q5_K] = {697 .type_name = "q5_K",698 .blck_size = QK_K,699 .type_size = sizeof(block_q5_K),700 .is_quantized = true,701 .to_float = (ggml_to_float_t) dequantize_row_q5_K,702 .from_float_ref = (ggml_from_float_t) quantize_row_q5_K_ref,703 },704 [GGML_TYPE_Q6_K] = {705 .type_name = "q6_K",706 .blck_size = QK_K,707 .type_size = sizeof(block_q6_K),708 .is_quantized = true,709 .to_float = (ggml_to_float_t) dequantize_row_q6_K,710 .from_float_ref = (ggml_from_float_t) quantize_row_q6_K_ref,711 },712 [GGML_TYPE_IQ2_XXS] = {713 .type_name = "iq2_xxs",714 .blck_size = QK_K,715 .type_size = sizeof(block_iq2_xxs),716 .is_quantized = true,717 .to_float = (ggml_to_float_t) dequantize_row_iq2_xxs,718 .from_float_ref = NULL,719 },720 [GGML_TYPE_IQ2_XS] = {721 .type_name = "iq2_xs",722 .blck_size = QK_K,723 .type_size = sizeof(block_iq2_xs),724 .is_quantized = true,725 .to_float = (ggml_to_float_t) dequantize_row_iq2_xs,726 .from_float_ref = NULL,727 },728 [GGML_TYPE_IQ3_XXS] = {729 .type_name = "iq3_xxs",730 .blck_size = QK_K,731 .type_size = sizeof(block_iq3_xxs),732 .is_quantized = true,733 .to_float = (ggml_to_float_t) dequantize_row_iq3_xxs,734 .from_float_ref = (ggml_from_float_t)quantize_row_iq3_xxs_ref,735 },736 [GGML_TYPE_IQ3_S] = {737 .type_name = "iq3_s",738 .blck_size = QK_K,739 .type_size = sizeof(block_iq3_s),740 .is_quantized = true,741 .to_float = (ggml_to_float_t) dequantize_row_iq3_s,742 .from_float_ref = (ggml_from_float_t)quantize_row_iq3_s_ref,743 },744 [GGML_TYPE_IQ2_S] = {745 .type_name = "iq2_s",746 .blck_size = QK_K,747 .type_size = sizeof(block_iq2_s),748 .is_quantized = true,749 .to_float = (ggml_to_float_t) dequantize_row_iq2_s,750 .from_float_ref = (ggml_from_float_t)quantize_row_iq2_s_ref,751 },752 [GGML_TYPE_IQ1_S] = {753 .type_name = "iq1_s",754 .blck_size = QK_K,755 .type_size = sizeof(block_iq1_s),756 .is_quantized = true,757 .to_float = (ggml_to_float_t) dequantize_row_iq1_s,758 .from_float_ref = NULL,759 },760 [GGML_TYPE_IQ1_M] = {761 .type_name = "iq1_m",762 .blck_size = QK_K,763 .type_size = sizeof(block_iq1_m),764 .is_quantized = true,765 .to_float = (ggml_to_float_t) dequantize_row_iq1_m,766 .from_float_ref = NULL,767 },768 [GGML_TYPE_IQ4_NL] = {769 .type_name = "iq4_nl",770 .blck_size = QK4_NL,771 .type_size = sizeof(block_iq4_nl),772 .is_quantized = true,773 .to_float = (ggml_to_float_t) dequantize_row_iq4_nl,774 .from_float_ref = (ggml_from_float_t)quantize_row_iq4_nl_ref,775 },776 [GGML_TYPE_IQ4_XS] = {777 .type_name = "iq4_xs",778 .blck_size = QK_K,779 .type_size = sizeof(block_iq4_xs),780 .is_quantized = true,781 .to_float = (ggml_to_float_t) dequantize_row_iq4_xs,782 .from_float_ref = (ggml_from_float_t)quantize_row_iq4_xs_ref,783 },784 [GGML_TYPE_Q8_K] = {785 .type_name = "q8_K",786 .blck_size = QK_K,787 .type_size = sizeof(block_q8_K),788 .is_quantized = true,789 },790 [GGML_TYPE_BF16] = {791 .type_name = "bf16",792 .blck_size = 1,793 .type_size = sizeof(ggml_bf16_t),794 .is_quantized = false,795 .to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,796 .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,797 },798 [31] = { // GGML_TYPE_Q4_0_4_4799 .type_name = "TYPE_Q4_0_4_4 REMOVED, use Q4_0 with runtime repacking",800 .blck_size = 0,801 .type_size = 0,802 .is_quantized = false,803 },804 [32] = { // GGML_TYPE_Q4_0_4_8805 .type_name = "TYPE_Q4_0_4_8 REMOVED, use Q4_0 with runtime repacking",806 .blck_size = 0,807 .type_size = 0,808 .is_quantized = false,809 },810 [33] = { // GGML_TYPE_Q4_0_8_8811 .type_name = "TYPE_Q4_0_8_8 REMOVED, use Q4_0 with runtime repacking",812 .blck_size = 0,813 .type_size = 0,814 .is_quantized = false,815 },816 [GGML_TYPE_TQ1_0] = {817 .type_name = "tq1_0",818 .blck_size = QK_K,819 .type_size = sizeof(block_tq1_0),820 .is_quantized = true,821 .to_float = (ggml_to_float_t) dequantize_row_tq1_0,822 .from_float_ref = (ggml_from_float_t) quantize_row_tq1_0_ref,823 },824 [GGML_TYPE_TQ2_0] = {825 .type_name = "tq2_0",826 .blck_size = QK_K,827 .type_size = sizeof(block_tq2_0),828 .is_quantized = true,829 .to_float = (ggml_to_float_t) dequantize_row_tq2_0,830 .from_float_ref = (ggml_from_float_t) quantize_row_tq2_0_ref,831 },832 [36] = { // GGML_TYPE_IQ4_NL_4_4833 .type_name = "TYPE_IQ4_NL_4_4 REMOVED, use IQ4_NL with runtime repacking",834 .blck_size = 0,835 .type_size = 0,836 .is_quantized = false,837 },838 [37] = { // GGML_TYPE_IQ4_NL_4_8839 .type_name = "TYPE_IQ4_NL_4_8 REMOVED, use IQ4_NL with runtime repacking",840 .blck_size = 0,841 .type_size = 0,842 .is_quantized = false,843 },844 [38] = { // GGML_TYPE_IQ4_NL_8_8845 .type_name = "TYPE_IQ4_NL_8_8 REMOVED, use IQ4_NL with runtime repacking",846 .blck_size = 0,847 .type_size = 0,848 .is_quantized = false,849 },850};851 852const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type) {853 GGML_ASSERT(type < GGML_TYPE_COUNT);854 return &type_traits[type];855}856 857//858// ggml object859//860 861struct ggml_object {862 size_t offs;863 size_t size;864 865 struct ggml_object * next;866 867 enum ggml_object_type type;868 869 char padding[4];870};871 872static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);873 874//875// ggml context876//877 878struct ggml_context {879 size_t mem_size;880 void * mem_buffer;881 bool mem_buffer_owned;882 bool no_alloc;883 884 int n_objects;885 886 struct ggml_object * objects_begin;887 struct ggml_object * objects_end;888};889 890struct ggml_context_container {891 bool used;892 893 struct ggml_context context;894};895 896//897// data types898//899 900static const char * GGML_OP_NAME[GGML_OP_COUNT] = {901 "NONE",902 903 "DUP",904 "ADD",905 "ADD1",906 "ACC",907 "SUB",908 "MUL",909 "DIV",910 "SQR",911 "SQRT",912 "LOG",913 "SIN",914 "COS",915 "SUM",916 "SUM_ROWS",917 "MEAN",918 "ARGMAX",919 "COUNT_EQUAL",920 "REPEAT",921 "REPEAT_BACK",922 "CONCAT",923 "SILU_BACK",924 "NORM",925 "RMS_NORM",926 "RMS_NORM_BACK",927 "GROUP_NORM",928 929 "MUL_MAT",930 "MUL_MAT_ID",931 "OUT_PROD",932 933 "SCALE",934 "SET",935 "CPY",936 "CONT",937 "RESHAPE",938 "VIEW",939 "PERMUTE",940 "TRANSPOSE",941 "GET_ROWS",942 "GET_ROWS_BACK",943 "DIAG",944 "DIAG_MASK_INF",945 "DIAG_MASK_ZERO",946 "SOFT_MAX",947 "SOFT_MAX_BACK",948 "ROPE",949 "ROPE_BACK",950 "CLAMP",951 "CONV_TRANSPOSE_1D",952 "IM2COL",953 "IM2COL_BACK",954 "CONV_TRANSPOSE_2D",955 "POOL_1D",956 "POOL_2D",957 "POOL_2D_BACK",958 "UPSCALE",959 "PAD",960 "PAD_REFLECT_1D",961 "ARANGE",962 "TIMESTEP_EMBEDDING",963 "ARGSORT",964 "LEAKY_RELU",965 966 "FLASH_ATTN_EXT",967 "FLASH_ATTN_BACK",968 "SSM_CONV",969 "SSM_SCAN",970 "WIN_PART",971 "WIN_UNPART",972 "GET_REL_POS",973 "ADD_REL_POS",974 "RWKV_WKV6",975 "GATED_LINEAR_ATTN",976 977 "UNARY",978 979 "MAP_UNARY",980 "MAP_BINARY",981 982 "MAP_CUSTOM1_F32",983 "MAP_CUSTOM2_F32",984 "MAP_CUSTOM3_F32",985 986 "MAP_CUSTOM1",987 "MAP_CUSTOM2",988 "MAP_CUSTOM3",989 990 "CROSS_ENTROPY_LOSS",991 "CROSS_ENTROPY_LOSS_BACK",992 "OPT_STEP_ADAMW",993};994 995static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83");996 997static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {998 "none",999 1000 "x",1001 "x+y",1002 "x+y",1003 "view(x,nb,offset)+=y->x",1004 "x-y",1005 "x*y",1006 "x/y",1007 "x^2",1008 "√x",1009 "log(x)",1010 "sin(x)",1011 "cos(x)",1012 "Σx",1013 "Σx_k",1014 "Σx/n",1015 "argmax(x)",1016 "count_equal(x)",1017 "repeat(x)",1018 "repeat_back(x)",1019 "concat(x, y)",1020 "silu_back(x)",1021 "norm(x)",1022 "rms_norm(x)",1023 "rms_norm_back(x)",1024 "group_norm(x)",1025 1026 "X*Y",1027 "X[i]*Y",1028 "X*Y",1029 1030 "x*v",1031 "y-\\>view(x)",1032 "x-\\>y",1033 "cont(x)",1034 "reshape(x)",1035 "view(x)",1036 "permute(x)",1037 "transpose(x)",1038 "get_rows(x)",1039 "get_rows_back(x)",1040 "diag(x)",1041 "diag_mask_inf(x)",1042 "diag_mask_zero(x)",1043 "soft_max(x)",1044 "soft_max_back(x)",1045 "rope(x)",1046 "rope_back(x)",1047 "clamp(x)",1048 "conv_transpose_1d(x)",1049 "im2col(x)",1050 "im2col_back(x)",1051 "conv_transpose_2d(x)",1052 "pool_1d(x)",1053 "pool_2d(x)",1054 "pool_2d_back(x)",1055 "upscale(x)",1056 "pad(x)",1057 "pad_reflect_1d(x)",1058 "arange(start, stop, step)",1059 "timestep_embedding(timesteps, dim, max_period)",1060 "argsort(x)",1061 "leaky_relu(x)",1062 1063 "flash_attn_ext(x)",1064 "flash_attn_back(x)",1065 "ssm_conv(x)",1066 "ssm_scan(x)",1067 "win_part(x)",1068 "win_unpart(x)",1069 "get_rel_pos(x)",1070 "add_rel_pos(x)",1071 "rwkv_wkv6(k, v, r, tf, td, s)",1072 "gated_linear_attn(k, v, q, gate, s)",1073 1074 "unary(x)",1075 1076 "f(x)",1077 "f(x,y)",1078 1079 "custom_f32(x)",1080 "custom_f32(x,y)",1081 "custom_f32(x,y,z)",1082 1083 "custom(x)",1084 "custom(x,y)",1085 "custom(x,y,z)",1086 1087 "cross_entropy_loss(x,y)",1088 "cross_entropy_loss_back(x,y)",1089 "adamw(x)",1090};1091 1092static_assert(GGML_OP_COUNT == 83, "GGML_OP_COUNT != 83");1093 1094static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");1095 1096 1097static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {1098 "ABS",1099 "SGN",1100 "NEG",1101 "STEP",1102 "TANH",1103 "ELU",1104 "RELU",1105 "SIGMOID",1106 "GELU",1107 "GELU_QUICK",1108 "SILU",1109 "HARDSWISH",1110 "HARDSIGMOID",1111 "EXP",1112};1113 1114static_assert(GGML_UNARY_OP_COUNT == 14, "GGML_UNARY_OP_COUNT != 14");1115 1116 1117static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");1118static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");1119 1120 1121////////////////////////////////////////////////////////////////////////////////1122 1123void ggml_print_object(const struct ggml_object * obj) {1124 GGML_LOG_INFO(" - ggml_object: type = %d, offset = %zu, size = %zu, next = %p\n",1125 obj->type, obj->offs, obj->size, (const void *) obj->next);1126}1127 1128void ggml_print_objects(const struct ggml_context * ctx) {1129 struct ggml_object * obj = ctx->objects_begin;1130 1131 GGML_LOG_INFO("%s: objects in context %p:\n", __func__, (const void *) ctx);1132 1133 while (obj != NULL) {1134 ggml_print_object(obj);1135 obj = obj->next;1136 }1137 1138 GGML_LOG_INFO("%s: --- end ---\n", __func__);1139}1140 1141int64_t ggml_nelements(const struct ggml_tensor * tensor) {1142 static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");1143 1144 return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];1145}1146 1147int64_t ggml_nrows(const struct ggml_tensor * tensor) {1148 static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");1149 1150 return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];1151}1152 1153size_t ggml_nbytes(const struct ggml_tensor * tensor) {1154 size_t nbytes;1155 const size_t blck_size = ggml_blck_size(tensor->type);1156 if (blck_size == 1) {1157 nbytes = ggml_type_size(tensor->type);1158 for (int i = 0; i < GGML_MAX_DIMS; ++i) {1159 nbytes += (tensor->ne[i] - 1)*tensor->nb[i];1160 }1161 }1162 else {1163 nbytes = tensor->ne[0]*tensor->nb[0]/blck_size;1164 for (int i = 1; i < GGML_MAX_DIMS; ++i) {1165 nbytes += (tensor->ne[i] - 1)*tensor->nb[i];1166 }1167 }1168 1169 return nbytes;1170}1171 1172size_t ggml_nbytes_pad(const struct ggml_tensor * tensor) {1173 return GGML_PAD(ggml_nbytes(tensor), GGML_MEM_ALIGN);1174}1175 1176int64_t ggml_blck_size(enum ggml_type type) {1177 return type_traits[type].blck_size;1178}1179 1180size_t ggml_type_size(enum ggml_type type) {1181 return type_traits[type].type_size;1182}1183 1184size_t ggml_row_size(enum ggml_type type, int64_t ne) {1185 assert(ne % ggml_blck_size(type) == 0);1186 return ggml_type_size(type)*ne/ggml_blck_size(type);1187}1188 1189double ggml_type_sizef(enum ggml_type type) {1190 return ((double)(type_traits[type].type_size))/type_traits[type].blck_size;1191}1192 1193const char * ggml_type_name(enum ggml_type type) {1194 return type < GGML_TYPE_COUNT ? type_traits[type].type_name : "NONE";1195}1196 1197bool ggml_is_quantized(enum ggml_type type) {1198 return type_traits[type].is_quantized;1199}1200