KBaba7/llama.cpp
0
1#pragma once2 3// GGML internal header4 5#include "ggml.h"6#include "gguf.h"7 8#include <assert.h>9#include <math.h>10#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/11#include <stdbool.h>12#include <stdint.h>13#include <string.h>14 15#ifdef __ARM_FEATURE_SVE16#include <arm_sve.h>17#endif // __ARM_FEATURE_SVE18 19#if defined(__ARM_NEON) && !defined(__CUDACC__)20// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:21//22// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/23//24#include <arm_neon.h>25#endif26 27#if defined(__F16C__)28#include <immintrin.h>29#endif30 31#ifdef __cplusplus32extern "C" {33#endif34 35#ifndef MIN36# define MIN(a, b) ((a) < (b) ? (a) : (b))37#endif38 39#ifndef MAX40# define MAX(a, b) ((a) > (b) ? (a) : (b))41#endif42 43// required for mmap as gguf only guarantees 32-byte alignment44#define TENSOR_ALIGNMENT 3245 46// static_assert should be a #define, but if it's not,47// fall back to the _Static_assert C11 keyword.48// if C99 - static_assert is noop49// ref: https://stackoverflow.com/a/53923785/403997650#ifndef __cplusplus51 #ifndef static_assert52 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)53 #define static_assert(cond, msg) _Static_assert(cond, msg)54 #else55 #define static_assert(cond, msg) struct global_scope_noop_trick56 #endif57 #endif58#endif59 60static inline int ggml_up32(int n) {61 return (n + 31) & ~31;62}63 64//static inline int ggml_up64(int n) {65// return (n + 63) & ~63;66//}67 68static inline int ggml_up(int n, int m) {69 // assert m is a power of 270 GGML_ASSERT((m & (m - 1)) == 0);71 return (n + m - 1) & ~(m - 1);72}73 74//75// logging76//77 78GGML_ATTRIBUTE_FORMAT(2, 3)79GGML_API void ggml_log_internal (enum ggml_log_level level, const char * format, ...);80GGML_API void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data);81 82#define GGML_LOG(...) ggml_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__)83#define GGML_LOG_INFO(...) ggml_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)84#define GGML_LOG_WARN(...) ggml_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)85#define GGML_LOG_ERROR(...) ggml_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)86#define GGML_LOG_DEBUG(...) ggml_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)87#define GGML_LOG_CONT(...) ggml_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__)88 89#define GGML_DEBUG 090 91#if (GGML_DEBUG >= 1)92#define GGML_PRINT_DEBUG(...) GGML_LOG_DEBUG(__VA_ARGS__)93#else94#define GGML_PRINT_DEBUG(...)95#endif96 97#if (GGML_DEBUG >= 5)98#define GGML_PRINT_DEBUG_5(...) GGML_LOG_DEBUG(__VA_ARGS__)99#else100#define GGML_PRINT_DEBUG_5(...)101#endif102 103#if (GGML_DEBUG >= 10)104#define GGML_PRINT_DEBUG_10(...) GGML_LOG_DEBUG(__VA_ARGS__)105#else106#define GGML_PRINT_DEBUG_10(...)107#endif108 109// tensor params110 111static void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) {112 GGML_ASSERT(tensor != NULL); // silence -Warray-bounds warnings113 assert(params_size <= GGML_MAX_OP_PARAMS);114 memcpy(tensor->op_params, params, params_size);115}116 117static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {118 assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));119 return ((const int32_t *)(tensor->op_params))[i];120}121 122static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) {123 assert(i < GGML_MAX_OP_PARAMS / sizeof(float));124 return ((const float *)(tensor->op_params))[i];125}126 127static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {128 assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));129 ((int32_t *)(tensor->op_params))[i] = value;130}131 132static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) {133 assert(i < GGML_MAX_OP_PARAMS / sizeof(float));134 ((float *)(tensor->op_params))[i] = value;135}136 137struct ggml_map_custom1_op_params {138 ggml_custom1_op_t fun;139 int n_tasks;140 void * userdata;141};142 143struct ggml_map_custom2_op_params {144 ggml_custom2_op_t fun;145 int n_tasks;146 void * userdata;147};148 149struct ggml_map_custom3_op_params {150 ggml_custom3_op_t fun;151 int n_tasks;152 void * userdata;153};154 155// bitset156 157typedef uint32_t ggml_bitset_t;158 159static_assert(sizeof(ggml_bitset_t) == 4, "bitset_t constants must be updated");160#define BITSET_SHR 5 // log2(sizeof(ggml_bitset_t)*8)161#define BITSET_MASK (sizeof(ggml_bitset_t)*8 - 1)162 163static size_t ggml_bitset_size(size_t n) {164 return (n + BITSET_MASK) >> BITSET_SHR;165}166 167static inline bool ggml_bitset_get(const ggml_bitset_t * bitset, size_t i) {168 return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK)));169}170 171static inline void ggml_bitset_set(ggml_bitset_t * bitset, size_t i) {172 bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK));173}174 175static inline void ggml_bitset_clear(ggml_bitset_t * bitset, size_t i) {176 bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK));177}178 179// hash set180 181#define GGML_HASHSET_FULL ((size_t)-1)182#define GGML_HASHSET_ALREADY_EXISTS ((size_t)-2)183 184struct ggml_hash_set {185 size_t size;186 ggml_bitset_t * used; // whether or not the keys are in use i.e. set187 struct ggml_tensor ** keys; // actual tensors in the set, keys[i] is only defined if ggml_bitset_get(used, i)188};189 190struct ggml_hash_set ggml_hash_set_new(size_t size);191void ggml_hash_set_free(struct ggml_hash_set * hash_set);192 193// returns the minimum size for a hash set that can hold min_sz elements194size_t ggml_hash_size(size_t min_sz);195 196// remove all elements from the hash set197void ggml_hash_set_reset(struct ggml_hash_set * hash_set);198 199// returns true if key is in the hash set200static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key);201 202// returns GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted203static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key);204 205// returns GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full206static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);207 208// return index, asserts if table is full209static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);210 211// hash function for ggml_tensor212static inline size_t ggml_hash(const struct ggml_tensor * p) {213 // the last 4 bits are always zero due to alignment214 return (size_t)(uintptr_t)p >> 4;215}216 217static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key) {218 size_t h = ggml_hash(key) % hash_set->size;219 220 // linear probing221 size_t i = h;222 while (ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) {223 i = (i + 1) % hash_set->size;224 if (i == h) {225 // visited all hash table entries -> not found226 return GGML_HASHSET_FULL;227 }228 }229 return i;230}231 232static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) {233 size_t i = ggml_hash_find(hash_set, key);234 return i != GGML_HASHSET_FULL && ggml_bitset_get(hash_set->used, i);235}236 237static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {238 size_t h = ggml_hash(key) % hash_set->size;239 240 // linear probing241 size_t i = h;242 do {243 if (!ggml_bitset_get(hash_set->used, i)) {244 ggml_bitset_set(hash_set->used, i);245 hash_set->keys[i] = key;246 return i;247 }248 if (hash_set->keys[i] == key) {249 return GGML_HASHSET_ALREADY_EXISTS;250 }251 i = (i + 1) % hash_set->size;252 } while (i != h);253 254 // visited all hash table entries -> not found255 GGML_ABORT("fatal error");256}257 258static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {259 size_t h = ggml_hash(key) % hash_set->size;260 261 // linear probing262 size_t i = h;263 do {264 if (!ggml_bitset_get(hash_set->used, i)) {265 ggml_bitset_set(hash_set->used, i);266 hash_set->keys[i] = key;267 return i;268 }269 if (hash_set->keys[i] == key) {270 return i;271 }272 i = (i + 1) % hash_set->size;273 } while (i != h);274 275 // visited all hash table entries -> not found276 GGML_ABORT("fatal error");277}278 279// computation graph280 281enum ggml_cgraph_eval_order {282 GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,283 GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,284 GGML_CGRAPH_EVAL_ORDER_COUNT285};286 287struct ggml_cgraph {288 int size; // maximum number of nodes/leafs/grads/grad_accs289 int n_nodes; // number of nodes currently in use290 int n_leafs; // number of leafs currently in use291 292 struct ggml_tensor ** nodes; // tensors with data that can change if the graph is evaluated293 struct ggml_tensor ** grads; // the outputs of these tensors are the gradients of the nodes294 struct ggml_tensor ** grad_accs; // accumulators for node gradients295 struct ggml_tensor ** leafs; // tensors with constant data296 297 struct ggml_hash_set visited_hash_set;298 299 enum ggml_cgraph_eval_order order;300};301 302// returns a slice of cgraph with nodes [i0, i1)303// the slice does not have leafs or gradients304// if you need the gradients, get them from the original graph305struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph, int i0, int i1);306 307// Memory allocation308 309GGML_API void * ggml_aligned_malloc(size_t size);310GGML_API void ggml_aligned_free(void * ptr, size_t size);311 312// FP16 to FP32 conversion313 314#if defined(__ARM_NEON)315 #if defined(_MSC_VER) || (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11)316 typedef uint16_t ggml_fp16_internal_t;317 #else318 typedef __fp16 ggml_fp16_internal_t;319 #endif320#endif321 322#if defined(__ARM_NEON) && !defined(_MSC_VER) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11)323 #define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)324 #define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)325 326 #define GGML_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)327 328 static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {329 ggml_fp16_internal_t tmp;330 memcpy(&tmp, &h, sizeof(ggml_fp16_t));331 return (float)tmp;332 }333 334 static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {335 ggml_fp16_t res;336 ggml_fp16_internal_t tmp = f;337 memcpy(&res, &tmp, sizeof(ggml_fp16_t));338 return res;339 }340 341#elif defined(__F16C__)342 343 #ifdef _MSC_VER344 #define GGML_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))345 #define GGML_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)346 #else347 #define GGML_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)348 #define GGML_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)349 #endif350 351#elif defined(__POWER9_VECTOR__)352 353 #define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)354 #define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)355 /* the inline asm below is about 12% faster than the lookup method */356 #define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)357 #define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)358 359 static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {360 register float f;361 register double d;362 __asm__(363 "mtfprd %0,%2\n"364 "xscvhpdp %0,%0\n"365 "frsp %1,%0\n" :366 /* temp */ "=d"(d),367 /* out */ "=f"(f):368 /* in */ "r"(h));369 return f;370 }371 372 static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {373 register double d;374 register ggml_fp16_t r;375 __asm__( /* xscvdphp can work on double or single precision */376 "xscvdphp %0,%2\n"377 "mffprd %1,%0\n" :378 /* temp */ "=d"(d),379 /* out */ "=r"(r):380 /* in */ "f"(f));381 return r;382 }383 384#else385 386 // FP16 <-> FP32387 // ref: https://github.com/Maratyszcza/FP16388 389 static inline float fp32_from_bits(uint32_t w) {390 union {391 uint32_t as_bits;392 float as_value;393 } fp32;394 fp32.as_bits = w;395 return fp32.as_value;396 }397 398 static inline uint32_t fp32_to_bits(float f) {399 union {400 float as_value;401 uint32_t as_bits;402 } fp32;403 fp32.as_value = f;404 return fp32.as_bits;405 }406 407 static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {408 const uint32_t w = (uint32_t) h << 16;409 const uint32_t sign = w & UINT32_C(0x80000000);410 const uint32_t two_w = w + w;411 412 const uint32_t exp_offset = UINT32_C(0xE0) << 23;413 #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)414 const float exp_scale = 0x1.0p-112f;415 #else416 const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));417 #endif418 const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;419 420 const uint32_t magic_mask = UINT32_C(126) << 23;421 const float magic_bias = 0.5f;422 const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;423 424 const uint32_t denormalized_cutoff = UINT32_C(1) << 27;425 const uint32_t result = sign |426 (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));427 return fp32_from_bits(result);428 }429 430 static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {431 #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)432 const float scale_to_inf = 0x1.0p+112f;433 const float scale_to_zero = 0x1.0p-110f;434 #else435 const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));436 const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));437 #endif438 float base = (fabsf(f) * scale_to_inf) * scale_to_zero;439 440 const uint32_t w = fp32_to_bits(f);441 const uint32_t shl1_w = w + w;442 const uint32_t sign = w & UINT32_C(0x80000000);443 uint32_t bias = shl1_w & UINT32_C(0xFF000000);444 if (bias < UINT32_C(0x71000000)) {445 bias = UINT32_C(0x71000000);446 }447 448 base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;449 const uint32_t bits = fp32_to_bits(base);450 const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);451 const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);452 const uint32_t nonsign = exp_bits + mantissa_bits;453 return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);454 }455 456 #define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)457 #define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)458 459#endif // defined(__ARM_NEON) && (!defined(__MSC_VER)460 461// precomputed f32 table for f16 (256 KB)462// defined in ggml.c, initialized in ggml_init()463GGML_API float ggml_table_f32_f16[1 << 16];464 465// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,466// so we define GGML_FP16_TO_FP32 and GGML_FP32_TO_FP16 elsewhere for NEON.467// This is also true for POWER9.468#if !defined(GGML_FP16_TO_FP32)469inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {470 uint16_t s;471 memcpy(&s, &f, sizeof(uint16_t));472 return ggml_table_f32_f16[s];473}474 475#define GGML_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)476#endif477 478#if !defined(GGML_FP32_TO_FP16)479#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)480#endif481 482/**483 * Converts brain16 to float32.484 *485 * The bfloat16 floating point format has the following structure:486 *487 * ┌sign488 * │489 * │ ┌exponent490 * │ │491 * │ │ ┌mantissa492 * │ │ │493 * │┌──┴───┐┌─┴───┐494 * 0b0000000000000000 brain16495 *496 * Since bf16 has the same number of exponent bits as a 32bit float,497 * encoding and decoding numbers becomes relatively straightforward.498 *499 * ┌sign500 * │501 * │ ┌exponent502 * │ │503 * │ │ ┌mantissa504 * │ │ │505 * │┌──┴───┐┌─┴───────────────────┐506 * 0b00000000000000000000000000000000 IEEE binary32507 *508 * For comparison, the standard fp16 format has fewer exponent bits.509 *510 * ┌sign511 * │512 * │ ┌exponent513 * │ │514 * │ │ ┌mantissa515 * │ │ │516 * │┌─┴─┐┌─┴──────┐517 * 0b0000000000000000 IEEE binary16518 *519 * @see IEEE 754-2008520 */521static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {522 union {523 float f;524 uint32_t i;525 } u;526 u.i = (uint32_t)h.bits << 16;527 return u.f;528}529 530/**531 * Converts float32 to brain16.532 *533 * This is binary identical with Google Brain float conversion.534 * Floats shall round to nearest even, and NANs shall be quiet.535 * Subnormals aren't flushed to zero, except perhaps when used.536 * This code should vectorize nicely if using modern compilers.537 */538static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {539 ggml_bf16_t h;540 union {541 float f;542 uint32_t i;543 } u;544 u.f = s;545 if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */546 h.bits = (u.i >> 16) | 64; /* force to quiet */547 return h;548 }549 h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;550 return h;551}552 553#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)554#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)555 556#ifdef __cplusplus557}558#endif559 560#ifdef __cplusplus561#include <vector>562 563// expose GGUF internals for test code564GGML_API size_t gguf_type_size(enum gguf_type type);565GGML_API struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params);566GGML_API void gguf_write_to_buf(const struct gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta);567#endif // __cplusplus568