KBaba7/llama.cpp
0
1#include "ggml-alloc.h"2#include "ggml-backend-impl.h"3#include "ggml.h"4#include "ggml-impl.h"5#include <assert.h>6#include <limits.h>7#include <stdarg.h>8#include <stdio.h>9#include <stdlib.h>10#include <string.h>11 12#define MAX(a, b) ((a) > (b) ? (a) : (b))13#define MAX_FREE_BLOCKS 25614 15//#define GGML_ALLOCATOR_DEBUG16 17//#define AT_PRINTF(...) GGML_LOG_DEBUG(__VA_ARGS__)18#define AT_PRINTF(...)19 20 21static bool ggml_is_view(const struct ggml_tensor * t) {22 return t->view_src != NULL;23}24 25static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {26 if (a->type != b->type) {27 return false;28 }29 for (int i = 0; i < GGML_MAX_DIMS; i++) {30 if (a->ne[i] != b->ne[i]) {31 return false;32 }33 if (a->nb[i] != b->nb[i]) {34 return false;35 }36 }37 return true;38}39 40// ops that return true for this function must not use restrict pointers for their backend implementations41static bool ggml_op_can_inplace(enum ggml_op op) {42 switch (op) {43 case GGML_OP_SCALE:44 case GGML_OP_DIAG_MASK_ZERO:45 case GGML_OP_DIAG_MASK_INF:46 case GGML_OP_ADD:47 case GGML_OP_ADD1:48 case GGML_OP_SUB:49 case GGML_OP_MUL:50 case GGML_OP_DIV:51 case GGML_OP_SQR:52 case GGML_OP_SQRT:53 case GGML_OP_LOG:54 case GGML_OP_UNARY:55 case GGML_OP_ROPE:56 case GGML_OP_ROPE_BACK:57 case GGML_OP_SILU_BACK:58 case GGML_OP_RMS_NORM:59 case GGML_OP_RMS_NORM_BACK:60 case GGML_OP_SOFT_MAX:61 case GGML_OP_SOFT_MAX_BACK:62 return true;63 64 default:65 return false;66 }67}68 69static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {70 assert(alignment && !(alignment & (alignment - 1))); // power of 271 size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;72 return offset + align;73}74 75// tallocr76 77struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) {78 void * base = ggml_backend_buffer_get_base(buffer);79 size_t align = ggml_backend_buffer_get_alignment(buffer);80 81 assert(align && !(align & (align - 1))); // power of 282 83 struct ggml_tallocr talloc = (struct ggml_tallocr) {84 /*.buffer = */ buffer,85 /*.base = */ base,86 /*.alignment = */ align,87 /*.offset = */ aligned_offset(base, 0, align),88 };89 return talloc;90}91 92void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) {93 size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);94 size = GGML_PAD(size, talloc->alignment);95 96 if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) {97 GGML_LOG_ERROR("%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\n",98 __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset);99 GGML_ABORT("not enough space in the buffer");100 }101 102 void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset;103 talloc->offset += size;104 105 assert(((uintptr_t)addr % talloc->alignment) == 0);106 107 ggml_backend_tensor_alloc(talloc->buffer, tensor, addr);108}109 110// dynamic tensor allocator111 112struct free_block {113 size_t offset;114 size_t size;115};116 117struct ggml_dyn_tallocr {118 size_t alignment;119 int n_free_blocks;120 struct free_block free_blocks[MAX_FREE_BLOCKS];121 size_t max_size;122 123#ifdef GGML_ALLOCATOR_DEBUG124 struct {125 const struct ggml_tensor * tensor;126 size_t offset;127 } allocated_tensors[1024];128#endif129};130 131#ifdef GGML_ALLOCATOR_DEBUG132static void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {133 for (int i = 0; i < 1024; i++) {134 if (alloc->allocated_tensors[i].tensor == NULL) {135 alloc->allocated_tensors[i].tensor = tensor;136 alloc->allocated_tensors[i].offset = offset;137 return;138 }139 }140 GGML_ABORT("out of allocated_tensors");141}142static void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {143 for (int i = 0; i < 1024; i++) {144 if (alloc->allocated_tensors[i].offset == offset) {145 alloc->allocated_tensors[i].tensor = NULL;146 return;147 }148 }149 GGML_ABORT("tried to free tensor %s not found\n", tensor->name);150}151#endif152 153static size_t ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {154 size = aligned_offset(NULL, size, alloc->alignment);155 156 AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size);157 158 size_t max_avail = 0;159 160 // find the best fitting free block besides the last block161 int best_fit_block = -1;162 size_t best_fit_size = SIZE_MAX;163 for (int i = 0; i < alloc->n_free_blocks - 1; i++) {164 struct free_block * block = &alloc->free_blocks[i];165 max_avail = MAX(max_avail, block->size);166 if (block->size >= size && block->size <= best_fit_size) {167 best_fit_block = i;168 best_fit_size = block->size;169 }170 }171 172 if (best_fit_block == -1) {173 // the last block is our last resort174 struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1];175 max_avail = MAX(max_avail, block->size);176 if (block->size >= size) {177 best_fit_block = alloc->n_free_blocks - 1;178 } else {179 // this should never happen180 GGML_LOG_ERROR("%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\n",181 __func__, size, max_avail);182 GGML_ABORT("not enough space in the buffer");183 }184 }185 186 struct free_block * block = &alloc->free_blocks[best_fit_block];187 size_t offset = block->offset;188 block->offset = offset + size;189 block->size -= size;190 if (block->size == 0) {191 // remove block if empty192 alloc->n_free_blocks--;193 for (int j = best_fit_block; j < alloc->n_free_blocks; j++) {194 alloc->free_blocks[j] = alloc->free_blocks[j+1];195 }196 }197 198 AT_PRINTF("block %d, offset %zu\n", best_fit_block, offset);199 200#ifdef GGML_ALLOCATOR_DEBUG201 add_allocated_tensor(alloc, offset, tensor);202 size_t cur_max = offset + size;203 if (cur_max > alloc->max_size) {204 // sort allocated_tensors by offset205 for (int i = 0; i < 1024; i++) {206 for (int j = i + 1; j < 1024; j++) {207 if (alloc->allocated_tensors[i].offset > alloc->allocated_tensors[j].offset) {208 const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor;209 size_t tmp_offset = alloc->allocated_tensors[i].offset;210 alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor;211 alloc->allocated_tensors[i].offset = alloc->allocated_tensors[j].offset;212 alloc->allocated_tensors[j].tensor = tmp_tensor;213 alloc->allocated_tensors[j].offset = tmp_offset;214 }215 }216 }217 GGML_LOG_DEBUG("max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);218 for (int i = 0; i < 1024; i++) {219 if (alloc->allocated_tensors[i].tensor) {220 GGML_LOG_DEBUG("%s [%zx-%zx] (%.2f MB) ", alloc->allocated_tensors[i].tensor->name,221 alloc->allocated_tensors[i].offset,222 alloc->allocated_tensors[i].offset + ggml_nbytes(alloc->allocated_tensors[i].tensor),223 ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0);224 }225 }226 GGML_LOG_DEBUG("\n");227 }228#endif229 230 alloc->max_size = MAX(alloc->max_size, offset + size);231 232 return offset;233 234 GGML_UNUSED(tensor);235}236 237// this is a very naive implementation, but for our case the number of free blocks should be very small238static void ggml_dyn_tallocr_free_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, size_t size, const struct ggml_tensor * tensor) {239 size = aligned_offset(NULL, size, alloc->alignment);240 241 AT_PRINTF("%s: freeing %s at %zu (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, offset, size, alloc->n_free_blocks);242 243#ifdef GGML_ALLOCATOR_DEBUG244 remove_allocated_tensor(alloc, offset, tensor);245#endif246 247 // see if we can merge with an existing block248 for (int i = 0; i < alloc->n_free_blocks; i++) {249 struct free_block * block = &alloc->free_blocks[i];250 // check if ptr is at the end of the block251 if (block->offset + block->size == offset) {252 block->size += size;253 // check if we can merge with the next block254 if (i < alloc->n_free_blocks - 1 && block->offset + block->size == alloc->free_blocks[i+1].offset) {255 block->size += alloc->free_blocks[i+1].size;256 alloc->n_free_blocks--;257 for (int j = i+1; j < alloc->n_free_blocks; j++) {258 alloc->free_blocks[j] = alloc->free_blocks[j+1];259 }260 }261 return;262 }263 // check if ptr is at the beginning of the block264 if (offset + size == block->offset) {265 block->offset = offset;266 block->size += size;267 // check if we can merge with the previous block268 if (i > 0 && alloc->free_blocks[i-1].offset + alloc->free_blocks[i-1].size == block->offset) {269 alloc->free_blocks[i-1].size += block->size;270 alloc->n_free_blocks--;271 for (int j = i; j < alloc->n_free_blocks; j++) {272 alloc->free_blocks[j] = alloc->free_blocks[j+1];273 }274 }275 return;276 }277 }278 // otherwise, add a new block279 GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks");280 // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)281 int insert_pos = 0;282 while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].offset < offset) {283 insert_pos++;284 }285 // shift all blocks from insert_pos onward to make room for the new block286 for (int i = alloc->n_free_blocks; i > insert_pos; i--) {287 alloc->free_blocks[i] = alloc->free_blocks[i-1];288 }289 // insert the new block290 alloc->free_blocks[insert_pos].offset = offset;291 alloc->free_blocks[insert_pos].size = size;292 alloc->n_free_blocks++;293 294 GGML_UNUSED(tensor);295}296 297static void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) {298 alloc->n_free_blocks = 1;299 alloc->free_blocks[0].offset = 0;300 alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows301 alloc->max_size = 0;302 303#ifdef GGML_ALLOCATOR_DEBUG304 for (int i = 0; i < 1024; i++) {305 alloc->allocated_tensors[i].tensor = NULL;306 }307#endif308}309 310static struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment) {311 struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr));312 313 *alloc = (struct ggml_dyn_tallocr) {314 /*.alignment = */ alignment,315 /*.n_free_blocks = */ 0,316 /*.free_blocks = */ {{0}},317 /*.max_size = */ 0,318#ifdef GGML_ALLOCATOR_DEBUG319 /*.allocated_tensors = */ {{0}},320#endif321 };322 323 ggml_dyn_tallocr_reset(alloc);324 325 return alloc;326}327 328static void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) {329 free(alloc);330}331 332static size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc) {333 return alloc->max_size;334}335 336 337/////////////////////////////////////338 339// graph allocator340 341struct hash_node {342 int n_children;343 int n_views;344 int buffer_id;345 size_t offset; // offset within the buffer346 bool allocated;347};348 349struct tensor_alloc {350 int buffer_id;351 size_t offset;352 size_t size_max; // 0 = pre-allocated, unused, or view353};354 355struct leaf_alloc {356 struct tensor_alloc leaf;357};358 359struct node_alloc {360 struct tensor_alloc dst;361 struct tensor_alloc src[GGML_MAX_SRC];362};363 364struct ggml_gallocr {365 ggml_backend_buffer_type_t * bufts; // [n_buffers]366 ggml_backend_buffer_t * buffers; // [n_buffers]367 struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers]368 int n_buffers;369 370 struct ggml_hash_set hash_set;371 struct hash_node * hash_values; // [hash_set.size]372 373 struct node_alloc * node_allocs; // [n_nodes]374 int n_nodes;375 376 struct leaf_alloc * leaf_allocs; // [n_leafs]377 int n_leafs;378};379 380ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) {381 ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(1, sizeof(struct ggml_gallocr));382 GGML_ASSERT(galloc != NULL);383 384 galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t));385 GGML_ASSERT(galloc->bufts != NULL);386 387 galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t));388 GGML_ASSERT(galloc->buffers != NULL);389 390 galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *));391 GGML_ASSERT(galloc->buf_tallocs != NULL);392 393 for (int i = 0; i < n_bufs; i++) {394 galloc->bufts[i] = bufts[i];395 galloc->buffers[i] = NULL;396 397 // check if the same buffer type is used multiple times and reuse the same allocator398 for (int j = 0; j < i; j++) {399 if (bufts[i] == bufts[j]) {400 galloc->buf_tallocs[i] = galloc->buf_tallocs[j];401 break;402 }403 }404 405 if (galloc->buf_tallocs[i] == NULL) {406 size_t alignment = ggml_backend_buft_get_alignment(bufts[i]);407 galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment);408 }409 }410 galloc->n_buffers = n_bufs;411 412 return galloc;413}414 415ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) {416 return ggml_gallocr_new_n(&buft, 1);417}418 419void ggml_gallocr_free(ggml_gallocr_t galloc) {420 if (galloc == NULL) {421 return;422 }423 424 for (int i = 0; i < galloc->n_buffers; i++) {425 if (galloc->buffers != NULL) {426 // skip if already freed427 bool freed = false;428 for (int j = 0; j < i; j++) {429 if (galloc->buffers[j] == galloc->buffers[i]) {430 freed = true;431 break;432 }433 }434 if (!freed) {435 ggml_backend_buffer_free(galloc->buffers[i]);436 }437 }438 if (galloc->buf_tallocs != NULL) {439 // skip if already freed440 bool freed = false;441 for (int j = 0; j < i; j++) {442 if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {443 freed = true;444 break;445 }446 }447 if (!freed) {448 ggml_dyn_tallocr_free(galloc->buf_tallocs[i]);449 }450 }451 }452 453 ggml_hash_set_free(&galloc->hash_set);454 free(galloc->hash_values);455 free(galloc->bufts);456 free(galloc->buffers);457 free(galloc->buf_tallocs);458 free(galloc->node_allocs);459 free(galloc->leaf_allocs);460 free(galloc);461}462 463typedef struct ggml_gallocr * ggml_gallocr_t;464 465static struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {466 size_t i = ggml_hash_find_or_insert(&galloc->hash_set, t);467 return &galloc->hash_values[i];468}469 470static bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) {471 return ggml_gallocr_hash_get(galloc, t)->allocated;472}473 474static bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) {475 return t->data != NULL || ggml_gallocr_hash_get(galloc, t)->allocated;476}477 478static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {479 GGML_ASSERT(buffer_id >= 0);480 struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);481 482 if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_is_view(node)) {483 hn->allocated = true;484 assert(hn->offset == 0);485 486 // try to reuse a parent's buffer (inplace)487 if (ggml_op_can_inplace(node->op)) {488 for (int i = 0; i < GGML_MAX_SRC; i++) {489 struct ggml_tensor * parent = node->src[i];490 if (parent == NULL) {491 continue;492 }493 494 // if the node's data is external, then we cannot re-use it495 if (!ggml_gallocr_is_own(galloc, parent)) {496 AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);497 continue;498 }499 500 // outputs cannot be reused501 if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) {502 AT_PRINTF("not reusing parent %s for %s as it is an output\n", parent->name, node->name);503 continue;504 }505 506 if (!ggml_are_same_layout(node, parent)) {507 AT_PRINTF("not reusing parent %s for %s as layouts are different\n", parent->name, node->name);508 continue;509 }510 511 struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);512 if (p_hn->n_children == 1 && p_hn->n_views == 0) {513 if (ggml_is_view(parent)) {514 struct ggml_tensor * view_src = parent->view_src;515 struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);516 if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {517 AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);518 assert(view_src_hn->offset == p_hn->offset);519 hn->buffer_id = p_hn->buffer_id;520 hn->offset = p_hn->offset;521 p_hn->allocated = false; // avoid freeing the parent522 view_src_hn->allocated = false;523 return;524 }525 } else {526 AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);527 hn->buffer_id = p_hn->buffer_id;528 hn->offset = p_hn->offset;529 p_hn->allocated = false; // avoid freeing the parent530 return;531 }532 }533 }534 }535 // allocate tensor from the buffer536 struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];537 ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];538 size_t size = ggml_backend_buft_get_alloc_size(buft, node);539 size_t offset = ggml_dyn_tallocr_alloc(alloc, size, node);540 hn->buffer_id = buffer_id;541 hn->offset = offset;542 }543}544 545static void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {546 // graph outputs are never freed547 if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {548 AT_PRINTF("not freeing output %s\n", node->name);549 return;550 }551 552 struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);553 size_t offset = hn->offset;554 int buffer_id = hn->buffer_id;555 struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];556 ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];557 size_t size = ggml_backend_buft_get_alloc_size(buft, node);558 ggml_dyn_tallocr_free_tensor(alloc, offset, size, node);559 hn->allocated = false;560}561 562static int get_node_buffer_id(const int * node_buffer_ids, int i) {563 return node_buffer_ids ? node_buffer_ids[i] : 0;564}565 566static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {567 // clear hash tables568 ggml_hash_set_reset(&galloc->hash_set);569 memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size);570 571 // allocate leafs572 // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes573 for (int i = 0; i < graph->n_leafs; i++) {574 struct ggml_tensor * leaf = graph->leafs[i];575 ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i));576 }577 578 // count number of children and views579 // allocate other graph inputs and leafs first to avoid overwriting them580 for (int i = 0; i < graph->n_nodes; i++) {581 struct ggml_tensor * node = graph->nodes[i];582 583 // TODO: better way to add external dependencies584 // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to585 // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node586 // itself is never used and should not be considered a dependency587 if (ggml_is_view(node) && node->op != GGML_OP_NONE) {588 struct ggml_tensor * view_src = node->view_src;589 ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;590 }591 592 if (node->flags & GGML_TENSOR_FLAG_INPUT) {593 ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i));594 }595 596 for (int j = 0; j < GGML_MAX_SRC; j++) {597 struct ggml_tensor * src = node->src[j];598 if (src == NULL) {599 continue;600 }601 602 ggml_gallocr_hash_get(galloc, src)->n_children += 1;603 604 // allocate explicit inputs605 if (src->flags & GGML_TENSOR_FLAG_INPUT) {606 ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));607 }608 }609 }610 611 // allocate tensors612 for (int i = 0; i < graph->n_nodes; i++) {613 struct ggml_tensor * node = graph->nodes[i];614 int buffer_id = get_node_buffer_id(node_buffer_ids, i);615 616 // allocate parents (only leafs need to be allocated at this point)617 for (int j = 0; j < GGML_MAX_SRC; j++) {618 struct ggml_tensor * parent = node->src[j];619 if (parent == NULL) {620 continue;621 }622 ggml_gallocr_allocate_node(galloc, parent, buffer_id);623 }624 625 // allocate node626 ggml_gallocr_allocate_node(galloc, node, buffer_id);627 628 AT_PRINTF("exec: %s (%s) <= ", ggml_op_desc(node), node->name);629 for (int j = 0; j < GGML_MAX_SRC; j++) {630 struct ggml_tensor * parent = node->src[j];631 if (parent == NULL) {632 continue;633 }634 AT_PRINTF("%s", parent->name);635 if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {636 AT_PRINTF(", ");637 }638 }639 AT_PRINTF("\n");640 641 // update parents642 for (int j = 0; j < GGML_MAX_SRC; j++) {643 struct ggml_tensor * parent = node->src[j];644 if (parent == NULL) {645 continue;646 }647 struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);648 p_hn->n_children -= 1;649 650 AT_PRINTF("parent %s: %d children, %d views, allocated: %d\n",651 parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);652 653 if (p_hn->n_children == 0 && p_hn->n_views == 0) {654 if (ggml_is_view(parent)) {655 struct ggml_tensor * view_src = parent->view_src;656 struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);657 view_src_hn->n_views -= 1;658 AT_PRINTF("view_src %s: %d children, %d views\n",659 view_src->name, view_src_hn->n_children, view_src_hn->n_views);660 if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) {661 ggml_gallocr_free_node(galloc, view_src);662 }663 }664 else if (p_hn->allocated) {665 ggml_gallocr_free_node(galloc, parent);666 }667 }668 AT_PRINTF("\n");669 }670 }671}672 673bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {674 size_t min_hash_size = graph->n_nodes + graph->n_leafs;675 // add 25% margin to avoid hash collisions676 min_hash_size += min_hash_size / 4;677 678 // initialize hash table679 if (galloc->hash_set.size < min_hash_size) {680 ggml_hash_set_free(&galloc->hash_set);681 galloc->hash_set = ggml_hash_set_new(min_hash_size);682 GGML_ASSERT(galloc->hash_set.keys != NULL);683 684 free(galloc->hash_values);685 galloc->hash_values = malloc(sizeof(struct hash_node) * galloc->hash_set.size);686 GGML_ASSERT(galloc->hash_values != NULL);687 }688 689 // reset allocators690 for (int i = 0; i < galloc->n_buffers; i++) {691 ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]);692 }693 694 // allocate in hash table695 ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids);696 697 // set the node_allocs from the hash table698 if (galloc->n_nodes < graph->n_nodes) {699 free(galloc->node_allocs);700 galloc->node_allocs = calloc(graph->n_nodes, sizeof(struct node_alloc));701 GGML_ASSERT(galloc->node_allocs != NULL);702 }703 galloc->n_nodes = graph->n_nodes;704 for (int i = 0; i < graph->n_nodes; i++) {705 struct ggml_tensor * node = graph->nodes[i];706 struct node_alloc * node_alloc = &galloc->node_allocs[i];707 if (node->view_src || node->data) {708 node_alloc->dst.buffer_id = -1;709 node_alloc->dst.offset = SIZE_MAX;710 node_alloc->dst.size_max = 0;711 } else {712 struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);713 node_alloc->dst.buffer_id = hn->buffer_id;714 node_alloc->dst.offset = hn->offset;715 node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);716 }717 for (int j = 0; j < GGML_MAX_SRC; j++) {718 struct ggml_tensor * src = node->src[j];719 if (!src || src->view_src || src->data) {720 node_alloc->src[j].buffer_id = -1;721 node_alloc->src[j].offset = SIZE_MAX;722 node_alloc->src[j].size_max = 0;723 } else {724 struct hash_node * hn = ggml_gallocr_hash_get(galloc, src);725 node_alloc->src[j].buffer_id = hn->buffer_id;726 node_alloc->src[j].offset = hn->offset;727 node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src);728 }729 }730 }731 if (galloc->n_leafs < graph->n_leafs) {732 free(galloc->leaf_allocs);733 galloc->leaf_allocs = calloc(graph->n_leafs, sizeof(galloc->leaf_allocs[0]));734 GGML_ASSERT(galloc->leaf_allocs != NULL);735 }736 galloc->n_leafs = graph->n_leafs;737 for (int i = 0; i < graph->n_leafs; i++) {738 struct ggml_tensor * leaf = graph->leafs[i];739 struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);740 if (leaf->view_src || leaf->data) {741 galloc->leaf_allocs[i].leaf.buffer_id = -1;742 galloc->leaf_allocs[i].leaf.offset = SIZE_MAX;743 galloc->leaf_allocs[i].leaf.size_max = 0;744 } else {745 galloc->leaf_allocs[i].leaf.buffer_id = hn->buffer_id;746 galloc->leaf_allocs[i].leaf.offset = hn->offset;747 galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);748 }749 }750 751 // reallocate buffers if needed752 for (int i = 0; i < galloc->n_buffers; i++) {753 // if the buffer type is used multiple times, we reuse the same buffer754 for (int j = 0; j < i; j++) {755 if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {756 galloc->buffers[i] = galloc->buffers[j];757 break;758 }759 }760 761 size_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0;762 size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]);763 764 // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views765 if (new_size > cur_size || galloc->buffers[i] == NULL) {766#ifndef NDEBUG767 GGML_LOG_DEBUG("%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);768#endif769 770 ggml_backend_buffer_free(galloc->buffers[i]);771 galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size);772 if (galloc->buffers[i] == NULL) {773 GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size);774 return false;775 }776 ggml_backend_buffer_set_usage(galloc->buffers[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE);777 }778 }779 780 return true;781}782 783bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {784 return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL);785}786 787static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, struct tensor_alloc * tensor_alloc) {788 int buffer_id = tensor_alloc->buffer_id;789 assert(tensor->data || tensor->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);790 791 if (tensor->view_src != NULL) {792 if (tensor->buffer == NULL) {793 assert(tensor_alloc->offset == SIZE_MAX);794 if (tensor->view_src->buffer == NULL) {795 // this tensor was allocated without ggml-backend796 return;797 }798 ggml_backend_view_init(tensor);799 }800 } else {801 if (tensor->data == NULL) {802 assert(tensor_alloc->offset != SIZE_MAX);803 assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);804 void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]);805 void * addr = (char *)base + tensor_alloc->offset;806 ggml_backend_tensor_alloc(galloc->buffers[buffer_id], tensor, addr);807 } else {808 if (tensor->buffer == NULL) {809 // this tensor was allocated without ggml-backend810 return;811 }812 }813 }814}815 816static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct tensor_alloc * talloc) {817 size_t node_size = 0;818 if (!node->data && !node->view_src) {819 GGML_ASSERT(talloc->buffer_id >= 0); // prevent segfault when misusing the API820 node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node);821 }822 return talloc->size_max >= node_size;823}824 825static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {826 if (galloc->n_nodes != graph->n_nodes) {827#ifndef NDEBUG828 GGML_LOG_DEBUG("%s: graph has different number of nodes\n", __func__);829#endif830 return true;831 }832 833 if (galloc->n_leafs != graph->n_leafs) {834#ifndef NDEBUG835 GGML_LOG_DEBUG("%s: graph has different number of leafs\n", __func__);836#endif837 return true;838 }839 840 for (int i = 0; i < graph->n_nodes; i++) {841 struct ggml_tensor * node = graph->nodes[i];842 struct node_alloc * node_alloc = &galloc->node_allocs[i];843 844 if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) {845#ifndef NDEBUG846 GGML_LOG_DEBUG("%s: node %s is not valid\n", __func__, node->name);847#endif848 return true;849 }850 851 for (int j = 0; j < GGML_MAX_SRC; j++) {852 struct ggml_tensor * src = node->src[j];853 if (src == NULL) {854 continue;855 }856 if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) {857#ifndef NDEBUG858 GGML_LOG_DEBUG("%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name);859#endif860 return true;861 }862 }863 }864 865 return false;866}867 868bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {869 if (ggml_gallocr_needs_realloc(galloc, graph)) {870 if (galloc->n_buffers == 1) {871#ifndef NDEBUG872 GGML_LOG_DEBUG("%s: reallocating buffers automatically\n", __func__);873#endif874 if (!ggml_gallocr_reserve(galloc, graph)) {875 return false;876 }877 } else {878#ifndef NDEBUG879 GGML_LOG_DEBUG("%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__);880#endif881 return false;882 }883 }884 885 // reset buffers886 for (int i = 0; i < galloc->n_buffers; i++) {887 if (galloc->buffers[i] != NULL) {888 ggml_backend_buffer_reset(galloc->buffers[i]);889 }890 }891 892 // allocate the graph tensors from the previous assignments893 // leafs894 for (int i = 0; i < graph->n_leafs; i++) {895 struct ggml_tensor * leaf = graph->leafs[i];896 struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i];897 ggml_gallocr_init_tensor(galloc, leaf, &leaf_alloc->leaf);898 }899 // nodes900 for (int i = 0; i < graph->n_nodes; i++) {901 struct ggml_tensor * node = graph->nodes[i];902 struct node_alloc * node_alloc = &galloc->node_allocs[i];903 for (int j = 0; j < GGML_MAX_SRC; j++) {904 struct ggml_tensor * src = node->src[j];905 if (src == NULL) {906 continue;907 }908 ggml_gallocr_init_tensor(galloc, src, &node_alloc->src[j]);909 }910 ggml_gallocr_init_tensor(galloc, node, &node_alloc->dst);911 }912 913 return true;914}915 916size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) {917 GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers);918 919 if (galloc->buffers[buffer_id] == NULL) {920 return 0;921 }922 923 for (int i = 0; i < buffer_id; i++) {924 if (galloc->buffers[i] == galloc->buffers[buffer_id]) {925 // this buffer is the same as a previous one due to the same buffer type being used multiple times926 // only return the buffer size the first time it appears to avoid double counting927 return 0;928 }929 }930 931 return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]);932}933 934// utils935 936static bool alloc_tensor_range(struct ggml_context * ctx,937 struct ggml_tensor * first, struct ggml_tensor * last,938 ggml_backend_buffer_type_t buft, size_t size,939 ggml_backend_buffer_t ** buffers, size_t * n_buffers) {940 ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);941 if (buffer == NULL) {942#ifndef NDEBUG943 GGML_LOG_DEBUG("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size);944#endif945 for (size_t i = 0; i < *n_buffers; i++) {946 ggml_backend_buffer_free((*buffers)[i]);947 }948 free(*buffers);949 return false;950 }951 952 struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);953 954 for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {955 if (t->data == NULL) {956 if (t->view_src == NULL) {957 ggml_tallocr_alloc(&tallocr, t);958 } else if (t->buffer == NULL) {959 ggml_backend_view_init(t);960 }961 } else {962 if (t->view_src != NULL && t->buffer == NULL) {963 // view of a pre-allocated tensor964 ggml_backend_view_init(t);965 }966 }967 }968 969 *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));970 (*buffers)[(*n_buffers)++] = buffer;971 972 return true;973}974 975ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {976 GGML_ASSERT(ggml_get_no_alloc(ctx) == true);977 978 size_t alignment = ggml_backend_buft_get_alignment(buft);979 size_t max_size = ggml_backend_buft_get_max_size(buft);980 981 ggml_backend_buffer_t * buffers = NULL;982 size_t n_buffers = 0;983 984 size_t cur_buf_size = 0;985 struct ggml_tensor * first = ggml_get_first_tensor(ctx);986 for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) {987 size_t this_size = 0;988 if (t->data == NULL && t->view_src == NULL) {989 this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);990 }991 992 if (cur_buf_size > 0 && (cur_buf_size + this_size) > max_size) {993 // allocate tensors in the current buffer994 if (!alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) {995 return NULL;996 }997 first = t;998 cur_buf_size = this_size;999 } else {1000 cur_buf_size += this_size;1001 }1002 }1003 1004 // allocate remaining tensors1005 if (cur_buf_size > 0) {1006 if (!alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) {1007 return NULL;1008 }1009 }1010 1011 if (n_buffers == 0) {1012#ifndef NDEBUG1013 GGML_LOG_DEBUG("%s: all tensors in the context are already allocated\n", __func__);1014#endif1015 return NULL;1016 }1017 1018 ggml_backend_buffer_t buffer;1019 if (n_buffers == 1) {1020 buffer = buffers[0];1021 } else {1022 buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers);1023 }1024 free(buffers);1025 return buffer;1026}1027 1028ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) {1029 return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend));1030}1031