Xenobd/whisper.cpp
0
1// Note: porting this file to C++ is a work in progress2 3#ifdef _WIN324#define WIN32_LEAN_AND_MEAN5#ifndef NOMINMAX6# define NOMINMAX7#endif8#include <windows.h>9#endif10 11#include "ggml-backend.h"12#include "ggml-backend-impl.h"13#include "ggml-alloc.h"14#include "ggml-impl.h"15 16#include <assert.h>17#include <limits.h>18#include <stdarg.h>19#include <stdio.h>20#include <stdlib.h>21#include <string.h>22#include <string>23#include <vector>24#include <algorithm>25 26#ifdef __APPLE__27#include <sys/types.h>28#include <sys/sysctl.h>29#endif30 31 32// backend buffer type33 34const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {35 return buft->iface.get_name(buft);36}37 38ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {39 if (size == 0) {40 // return a dummy buffer for zero-sized allocations41 return ggml_backend_buffer_init(buft, {}, NULL, 0);42 }43 44 return buft->iface.alloc_buffer(buft, size);45}46 47size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {48 return buft->iface.get_alignment(buft);49}50 51size_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {52 // get_max_size is optional, defaults to SIZE_MAX53 if (buft->iface.get_max_size) {54 return buft->iface.get_max_size(buft);55 }56 return SIZE_MAX;57}58 59size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {60 // get_alloc_size is optional, defaults to ggml_nbytes61 if (buft->iface.get_alloc_size) {62 size_t size = buft->iface.get_alloc_size(buft, tensor);63 assert(size >= ggml_nbytes(tensor));64 return size;65 }66 return ggml_nbytes(tensor);67}68 69bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {70 if (buft->iface.is_host) {71 return buft->iface.is_host(buft);72 }73 return false;74}75 76ggml_backend_dev_t ggml_backend_buft_get_device(ggml_backend_buffer_type_t buft) {77 return buft->device;78}79 80// backend buffer81 82ggml_backend_buffer_t ggml_backend_buffer_init(83 ggml_backend_buffer_type_t buft,84 struct ggml_backend_buffer_i iface,85 void * context,86 size_t size) {87 ggml_backend_buffer_t buffer = new ggml_backend_buffer {88 /* .interface = */ iface,89 /* .buft = */ buft,90 /* .context = */ context,91 /* .size = */ size,92 /* .usage = */ GGML_BACKEND_BUFFER_USAGE_ANY93 };94 95 return buffer;96}97 98const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {99 return ggml_backend_buft_name(ggml_backend_buffer_get_type(buffer));100}101 102void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {103 if (buffer == NULL) {104 return;105 }106 107 if (buffer->iface.free_buffer != NULL) {108 buffer->iface.free_buffer(buffer);109 }110 delete buffer;111}112 113size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {114 return buffer->size;115}116 117void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {118 // get_base is optional if the buffer is zero-sized119 if (buffer->size == 0) {120 return NULL;121 }122 123 void * base = buffer->iface.get_base(buffer);124 125 GGML_ASSERT(base != NULL && "backend buffer base cannot be NULL");126 127 return base;128}129 130enum ggml_status ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {131 // init_tensor is optional132 if (buffer->iface.init_tensor) {133 return buffer->iface.init_tensor(buffer, tensor);134 }135 return GGML_STATUS_SUCCESS;136}137 138void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {139 // clear is optional if the buffer is zero-sized140 if (buffer->size == 0) {141 return;142 }143 144 buffer->iface.clear(buffer, value);145}146 147size_t ggml_backend_buffer_get_alignment(ggml_backend_buffer_t buffer) {148 return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));149}150 151size_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {152 return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));153}154 155size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor) {156 return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);157}158 159bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {160 return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));161}162 163void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {164 buffer->usage = usage;165 166 // FIXME: add a generic callback to the buffer interface167 if (ggml_backend_buffer_is_multi_buffer(buffer)) {168 ggml_backend_multi_buffer_set_usage(buffer, usage);169 }170}171 172enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t buffer) {173 return buffer->usage;174}175 176ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {177 return buffer->buft;178}179 180void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {181 if (buffer->iface.reset) {182 buffer->iface.reset(buffer);183 }184}185 186bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {187 ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;188 if (dst_buf->iface.cpy_tensor) {189 return dst_buf->iface.cpy_tensor(dst_buf, src, dst);190 }191 return false;192}193 194// backend195 196ggml_guid_t ggml_backend_guid(ggml_backend_t backend) {197 if (backend == NULL) {198 return NULL;199 }200 return backend->guid;201}202 203const char * ggml_backend_name(ggml_backend_t backend) {204 if (backend == NULL) {205 return "NULL";206 }207 return backend->iface.get_name(backend);208}209 210void ggml_backend_free(ggml_backend_t backend) {211 if (backend == NULL) {212 return;213 }214 215 backend->iface.free(backend);216}217 218ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {219 return ggml_backend_dev_buffer_type(backend->device);220}221 222ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {223 return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);224}225 226size_t ggml_backend_get_alignment(ggml_backend_t backend) {227 return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));228}229 230size_t ggml_backend_get_max_size(ggml_backend_t backend) {231 return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));232}233 234void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {235 GGML_ASSERT(tensor->data != NULL && "tensor not allocated");236 GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");237 238 if (backend->iface.set_tensor_async == NULL) {239 ggml_backend_tensor_set(tensor, data, offset, size);240 } else {241 backend->iface.set_tensor_async(backend, tensor, data, offset, size);242 }243}244 245void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {246 GGML_ASSERT(tensor->data != NULL && "tensor not allocated");247 GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");248 249 if (backend->iface.get_tensor_async == NULL) {250 ggml_backend_tensor_get(tensor, data, offset, size);251 } else {252 backend->iface.get_tensor_async(backend, tensor, data, offset, size);253 }254}255 256void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {257 GGML_ASSERT(tensor);258 ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;259 260 if (size == 0) {261 return;262 }263 264 GGML_ASSERT(buf != NULL && "tensor buffer not set");265 GGML_ASSERT(tensor->data != NULL && "tensor not allocated");266 GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");267 268 buf->iface.set_tensor(buf, tensor, data, offset, size);269}270 271void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {272 GGML_ASSERT(tensor);273 ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;274 275 if (size == 0) {276 return;277 }278 279 GGML_ASSERT(buf != NULL && "tensor buffer not set");280 GGML_ASSERT(tensor->data != NULL && "tensor not allocated");281 GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");282 283 buf->iface.get_tensor(buf, tensor, data, offset, size);284}285 286void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {287 ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;288 289 if (size == 0) {290 return;291 }292 293 GGML_ASSERT(buf != NULL && "tensor buffer not set");294 GGML_ASSERT(tensor->data != NULL && "tensor not allocated");295 GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");296 GGML_ASSERT(buf->iface.memset_tensor != NULL && "memset not implemented by backend buffer");297 298 buf->iface.memset_tensor(buf, tensor, value, offset, size);299}300 301void ggml_backend_synchronize(ggml_backend_t backend) {302 if (backend->iface.synchronize == NULL) {303 return;304 }305 306 backend->iface.synchronize(backend);307}308 309ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {310 GGML_ASSERT(backend->iface.graph_plan_create != NULL);311 312 return backend->iface.graph_plan_create(backend, cgraph);313}314 315void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {316 GGML_ASSERT(backend->iface.graph_plan_free != NULL);317 318 backend->iface.graph_plan_free(backend, plan);319}320 321enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {322 GGML_ASSERT(backend->iface.graph_plan_compute != NULL);323 324 return backend->iface.graph_plan_compute(backend, plan);325}326 327enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {328 enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);329 ggml_backend_synchronize(backend);330 return err;331}332 333enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {334 return backend->iface.graph_compute(backend, cgraph);335}336 337bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {338 return ggml_backend_dev_supports_op(backend->device, op);339}340 341bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {342 return ggml_backend_dev_supports_buft(backend->device, buft);343}344 345bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {346 return ggml_backend_dev_offload_op(backend->device, op);347}348 349ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend) {350 return backend->device;351}352 353// backend copy354 355void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) {356 GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");357 358 if (src == dst) {359 return;360 }361 362 if (ggml_backend_buffer_is_host(src->buffer)) {363 ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));364 } else if (ggml_backend_buffer_is_host(dst->buffer)) {365 ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));366 } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {367#ifndef NDEBUG368 GGML_LOG_DEBUG("%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));369#endif370 size_t nbytes = ggml_nbytes(src);371 void * data = malloc(nbytes);372 ggml_backend_tensor_get(src, data, 0, nbytes);373 ggml_backend_tensor_set(dst, data, 0, nbytes);374 free(data);375 }376}377 378void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst) {379 GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");380 381 if (src == dst) {382 return;383 }384 385 if (backend_dst->iface.cpy_tensor_async != NULL) {386 if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {387 return;388 }389 }390 391 // an async copy would normally happen after all the queued operations on both backends are completed392 // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy393 ggml_backend_synchronize(backend_src);394 ggml_backend_synchronize(backend_dst);395 ggml_backend_tensor_copy(src, dst);396}397 398// events399 400ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device) {401 // null device is allowed for the transition period to the device interface402 if (device == NULL || device->iface.event_new == NULL) {403 return NULL;404 }405 return device->iface.event_new(device);406}407 408void ggml_backend_event_free(ggml_backend_event_t event) {409 if (event == NULL) {410 return;411 }412 event->device->iface.event_free(event->device, event);413}414 415void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend) {416 GGML_ASSERT(backend->iface.event_record != NULL);417 418 backend->iface.event_record(backend, event);419}420 421void ggml_backend_event_synchronize(ggml_backend_event_t event) {422 GGML_ASSERT(event->device->iface.event_synchronize);423 424 event->device->iface.event_synchronize(event->device, event);425}426 427void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {428 GGML_ASSERT(backend->iface.event_wait != NULL);429 430 backend->iface.event_wait(backend, event);431}432 433// Backend device434 435const char * ggml_backend_dev_name(ggml_backend_dev_t device) {436 return device->iface.get_name(device);437}438 439const char * ggml_backend_dev_description(ggml_backend_dev_t device) {440 return device->iface.get_description(device);441}442 443void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total) {444 device->iface.get_memory(device, free, total);445}446 447enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device) {448 return device->iface.get_type(device);449}450 451void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props) {452 memset(props, 0, sizeof(*props));453 device->iface.get_props(device, props);454}455 456ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device) {457 return device->reg;458}459 460ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params) {461 return device->iface.init_backend(device, params);462}463 464ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device) {465 return device->iface.get_buffer_type(device);466}467 468ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device) {469 if (device->iface.get_host_buffer_type == NULL) {470 return NULL;471 }472 473 return device->iface.get_host_buffer_type(device);474}475 476ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size) {477 return device->iface.buffer_from_host_ptr(device, ptr, size, max_tensor_size);478}479 480bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {481 return device->iface.supports_op(device, op);482}483 484bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {485 return device->iface.supports_buft(device, buft);486}487 488bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {489 if (device->iface.offload_op != NULL) {490 return device->iface.offload_op(device, op);491 }492 493 return false;494}495 496// Backend (reg)497 498const char * ggml_backend_reg_name(ggml_backend_reg_t reg) {499 return reg->iface.get_name(reg);500}501 502size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg) {503 return reg->iface.get_device_count(reg);504}505 506ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index) {507 return reg->iface.get_device(reg, index);508}509 510void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {511 if (!reg->iface.get_proc_address) {512 return NULL;513 }514 return reg->iface.get_proc_address(reg, name);515}516 517// multi-buffer buffer518 519struct ggml_backend_multi_buffer_context {520 ggml_backend_buffer_t * buffers;521 size_t n_buffers;522};523 524static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {525 ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;526 for (size_t i = 0; i < ctx->n_buffers; i++) {527 ggml_backend_buffer_free(ctx->buffers[i]);528 }529 530 free(ctx->buffers);531 free(ctx);532}533 534static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {535 ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;536 for (size_t i = 0; i < ctx->n_buffers; i++) {537 ggml_backend_buffer_clear(ctx->buffers[i], value);538 }539}540 541static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {542 /* .free_buffer = */ ggml_backend_multi_buffer_free_buffer,543 /* .get_base = */ NULL,544 /* .init_tensor = */ NULL,545 /* .memset_tensor = */ NULL,546 /* .set_tensor = */ NULL,547 /* .get_tensor = */ NULL,548 /* .cpy_tensor = */ NULL,549 /* .clear = */ ggml_backend_multi_buffer_clear,550 /* .reset = */ NULL,551};552 553ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {554 ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) malloc(sizeof(struct ggml_backend_multi_buffer_context));555 ctx->n_buffers = n_buffers;556 ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));557 558 GGML_ASSERT(ctx->buffers != NULL);559 560 size_t total_size = 0;561 for (size_t i = 0; i < n_buffers; i++) {562 ctx->buffers[i] = buffers[i];563 total_size += ggml_backend_buffer_get_size(buffers[i]);564 }565 566 return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_i, ctx, total_size);567}568 569bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {570 return buffer->iface.free_buffer == ggml_backend_multi_buffer_free_buffer;571}572 573void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {574 GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));575 ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;576 for (size_t i = 0; i < ctx->n_buffers; i++) {577 ggml_backend_buffer_set_usage(ctx->buffers[i], usage);578 }579}580 581// creates a copy of the tensor with the same memory layout582static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {583 struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);584 for (int i = 0; i < GGML_MAX_DIMS; i++) {585 dup->nb[i] = tensor->nb[i];586 }587 return dup;588}589 590static bool ggml_is_view_op(enum ggml_op op) {591 return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;592}593 594// scheduler595 596#ifndef GGML_SCHED_MAX_BACKENDS597#define GGML_SCHED_MAX_BACKENDS 16598#endif599 600#ifndef GGML_SCHED_MAX_SPLIT_INPUTS601#define GGML_SCHED_MAX_SPLIT_INPUTS GGML_MAX_SRC602#endif603 604#ifndef GGML_SCHED_MAX_COPIES605#define GGML_SCHED_MAX_COPIES 4606#endif607 608struct ggml_backend_sched_split {609 int backend_id;610 int i_start;611 int i_end;612 struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];613 int n_inputs;614 // graph view of this split615 struct ggml_cgraph graph;616};617 618struct ggml_backend_sched {619 bool is_reset; // true if the scheduler has been reset since the last graph split620 bool is_alloc;621 622 int n_backends;623 624 ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];625 ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];626 ggml_gallocr_t galloc;627 628 // hash map of the nodes in the graph629 struct ggml_hash_set hash_set;630 int * hv_tensor_backend_ids; // [hash_set.size]631 struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies]632 633 int * node_backend_ids; // [graph_size]634 int * leaf_backend_ids; // [graph_size]635 636 int * prev_node_backend_ids; // [graph_size]637 int * prev_leaf_backend_ids; // [graph_size]638 639 // copy of the graph with modified inputs640 struct ggml_cgraph graph;641 642 // graph splits643 struct ggml_backend_sched_split * splits;644 int n_splits;645 int splits_capacity;646 647 // pipeline parallelism support648 int n_copies;649 int cur_copy;650 int next_copy;651 ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];652 struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];653 int n_graph_inputs;654 655 struct ggml_context * ctx;656 657 ggml_backend_sched_eval_callback callback_eval;658 void * callback_eval_user_data;659 660 char * context_buffer;661 size_t context_buffer_size;662 663 bool op_offload;664 665 int debug;666};667 668#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)669#define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]670#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]671#define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)672 673// returns the priority of the backend, lower id is higher priority674static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {675 for (int i = 0; i < sched->n_backends; i++) {676 if (sched->backends[i] == backend) {677 return i;678 }679 }680 return -1;681}682 683static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor, const struct ggml_tensor * op) {684 ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;685 if (buffer == NULL) {686 return -1;687 }688 689 // find highest prio backend that supports the buffer type and the op690 for (int i = 0; i < sched->n_backends; i++) {691 if (ggml_backend_supports_buft(sched->backends[i], buffer->buft) &&692 ggml_backend_supports_op(sched->backends[i], op)) {693 return i;694 }695 }696 697#ifndef NDEBUG698 GGML_LOG_DEBUG("%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\n",699 __func__, ggml_op_desc(tensor), ggml_backend_buffer_name(buffer), tensor->name);700#endif701 702 return -1;703}704 705#if 0706#define GGML_SCHED_MAX_SPLITS_DEBUG 4096707static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only708#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)709#define GET_CAUSE(node) causes[hash_id(node)]710#else711#define SET_CAUSE(node, ...)712#define GET_CAUSE(node) ""713#endif714 715// returns the backend that should be used for the node based on the current locations716static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {717 // assign pre-allocated nodes to their backend718 int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);719 if (cur_backend_id != -1) {720 SET_CAUSE(tensor, "1.dst");721 return cur_backend_id;722 }723 724 // view_src725 if (tensor->view_src != NULL) {726 cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src, tensor);727 if (cur_backend_id != -1) {728 SET_CAUSE(tensor, "1.vsrc");729 return cur_backend_id;730 }731 }732 733 if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {734 // since the tensor is pre-allocated, it cannot be moved to another backend735 ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;736 GGML_ABORT("pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op));737 }738 739 // graph input740 if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {741 cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)742 SET_CAUSE(tensor, "1.inp");743 return cur_backend_id;744 }745 746 // operations with weights are preferably run on the same backend as the weights747 for (int i = 0; i < GGML_MAX_SRC; i++) {748 const struct ggml_tensor * src = tensor->src[i];749 if (src == NULL) {750 continue;751 }752 // skip ROPE since the rope freqs tensor is too small to choose a backend based on it753 // not an ideal solution754 if (tensor->op != GGML_OP_ROPE && src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {755 int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor);756 // check if a backend with higher prio wants to offload the op757 if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) {758 for (int b = 0; b < src_backend_id; b++) {759 if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) {760 SET_CAUSE(tensor, "1.off");761 return b;762 }763 }764 }765 SET_CAUSE(tensor, "1.wgt%d", i);766 return src_backend_id;767 }768 }769 770 return -1;771}772 773static char * fmt_size(size_t size) {774 static char buffer[128];775 if (size >= 1024*1024) {776 snprintf(buffer, sizeof(buffer), "%zuM", size/1024/1024);777 } else {778 snprintf(buffer, sizeof(buffer), "%zuK", size/1024);779 }780 return buffer;781}782 783static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {784 int cur_split = 0;785 for (int i = 0; i < graph->n_nodes; i++) {786 if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {787 ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];788 GGML_LOG_DEBUG("\n## SPLIT #%d: %s # %d inputs", cur_split, ggml_backend_name(split_backend),789 sched->splits[cur_split].n_inputs);790 for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {791 if (j == 0) {792 GGML_LOG_DEBUG(": ");793 }794 GGML_LOG_DEBUG("[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,795 fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));796 }797 GGML_LOG_DEBUG("\n");798 cur_split++;799 }800 struct ggml_tensor * node = graph->nodes[i];801 if (ggml_is_view_op(node->op)) {802 continue;803 }804 if (sched->debug > 1) {805 ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);806 GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d:", i, ggml_op_name(node->op), node->name,807 fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node),808 graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)]);809 for (int j = 0; j < GGML_MAX_SRC; j++) {810 struct ggml_tensor * src = node->src[j];811 if (src == NULL) {812 continue;813 }814 ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);815 GGML_LOG_DEBUG(" %20.20s (%5.5s) [%5.5s %8.8s]", src->name,816 fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));817 }818 GGML_LOG_DEBUG("\n");819 }820 }821}822 823static bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) {824 ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer;825 ggml_backend_buffer_type_t buft = NULL;826 827 if (buf) {828 // the tensor is already allocated829 buft = buf->buft;830 } else {831 // see if the tensor already has a backend assigned, and use the buffer type of that backend832 int tensor_backend_id = tensor_backend_id(t);833 if (tensor_backend_id == -1 && t->view_src) {834 tensor_backend_id = tensor_backend_id(t->view_src);835 }836 if (tensor_backend_id != -1) {837 buft = sched->bufts[tensor_backend_id];838 }839 }840 841 return buft != NULL && ggml_backend_supports_buft(sched->backends[backend_id], buft);842}843 844static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, struct ggml_tensor * node, int cur_backend_id, int * node_backend_id) {845 if (ggml_backend_supports_op(sched->backends[cur_backend_id], node)) {846 *node_backend_id = cur_backend_id;847 SET_CAUSE(node, "2.sup");848 }849}850 851// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend852static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {853 // reset splits854 sched->n_splits = 0;855 sched->n_graph_inputs = 0;856 sched->is_reset = false;857 858 struct ggml_init_params params = {859 /* .mem_size = */ sched->context_buffer_size,860 /* .mem_buffer = */ sched->context_buffer,861 /* .no_alloc = */ true862 };863 864 ggml_free(sched->ctx);865 866 sched->ctx = ggml_init(params);867 if (sched->ctx == NULL) {868 GGML_ABORT("%s: failed to initialize context\n", __func__);869 }870 871 // pass 1: assign backends to ops with pre-allocated inputs872 for (int i = 0; i < graph->n_leafs; i++) {873 struct ggml_tensor * leaf = graph->leafs[i];874 int * leaf_backend_id = &tensor_backend_id(leaf);875 // do not overwrite user assignments876 if (*leaf_backend_id == -1) {877 *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);878 }879 }880 881 for (int i = 0; i < graph->n_nodes; i++) {882 struct ggml_tensor * node = graph->nodes[i];883 int * node_backend_id = &tensor_backend_id(node);884 // do not overwrite user assignments885 if (*node_backend_id == -1) {886 *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);887 888#if 0889 // src890 if (node->op == GGML_OP_NONE) {891 continue;892 }893 894 for (int j = 0; j < GGML_MAX_SRC; j++) {895 struct ggml_tensor * src = node->src[j];896 if (src == NULL) {897 continue;898 }899 int * src_backend_id = &tensor_backend_id(src);900 if (*src_backend_id == -1) {901 *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);902 }903 }904#endif905 }906 }907 908 // pass 2: expand current backend assignments909 // assign the same backend to adjacent nodes910 // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)911 // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops912 // ops unsupported by the backend being expanded will be left unassigned so that they can be assigned later when the locations of its inputs are known913 // expand gpu down914 {915 int cur_backend_id = -1;916 for (int i = 0; i < graph->n_nodes; i++) {917 struct ggml_tensor * node = graph->nodes[i];918 if (ggml_is_view_op(node->op)) {919 continue;920 }921 int * node_backend_id = &tensor_backend_id(node);922 if (*node_backend_id != -1) {923 if (*node_backend_id == sched->n_backends - 1) {924 // skip cpu (lowest prio backend)925 cur_backend_id = -1;926 } else {927 cur_backend_id = *node_backend_id;928 }929 } else if (cur_backend_id != -1) {930 ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);931 }932 }933 }934 // expand gpu up935 {936 int cur_backend_id = -1;937 for (int i = graph->n_nodes - 1; i >= 0; i--) {938 struct ggml_tensor * node = graph->nodes[i];939 if (ggml_is_view_op(node->op)) {940 continue;941 }942 int * node_backend_id = &tensor_backend_id(node);943 if (*node_backend_id != -1) {944 if (*node_backend_id == sched->n_backends - 1) {945 // skip cpu (lowest prio backend)946 cur_backend_id = -1;947 } else {948 cur_backend_id = *node_backend_id;949 }950 } else if (cur_backend_id != -1) {951 ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);952 }953 }954 }955 // expand rest down956 {957 int cur_backend_id = -1;958 for (int i = 0; i < graph->n_nodes; i++) {959 struct ggml_tensor * node = graph->nodes[i];960 if (ggml_is_view_op(node->op)) {961 continue;962 }963 int * node_backend_id = &tensor_backend_id(node);964 if (*node_backend_id != -1) {965 cur_backend_id = *node_backend_id;966 } else if (cur_backend_id != -1) {967 ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);968 }969 }970 }971 // expand rest up972 {973 int cur_backend_id = -1;974 for (int i = graph->n_nodes - 1; i >= 0; i--) {975 struct ggml_tensor * node = graph->nodes[i];976 if (ggml_is_view_op(node->op)) {977 continue;978 }979 int * node_backend_id = &tensor_backend_id(node);980 if (*node_backend_id != -1) {981 cur_backend_id = *node_backend_id;982 } else if (cur_backend_id != -1) {983 ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);984 }985 }986 }987 988 // pass 3: upgrade nodes to higher prio backends with compatible buffer types989 // if the tensor is already in the same buffer type (*) as another higher priority backend, we should move it there990 // however, we also need to verify that the sources are in compatible buffer types991 // (*) the actual requirement is more relaxed, the buffer type of the backend should be supported by all the users of this tensor further down the graph992 // however, this is slow to verify, so we have a more strict requirement that the buffer type is the same993 // this is not uncommon since multiple backends can use host memory, with the same buffer type (eg. BLAS and CPU)994 // additionally, set remaining unassigned nodes to the backend with the most supported inputs995 // only nodes that could not be assigned during expansion due to the backend not supporting the op should be unassigned at this point996 for (int i = 0; i < graph->n_nodes; i++) {997 struct ggml_tensor * node = graph->nodes[i];998 if (ggml_is_view_op(node->op)) {999 continue;1000 }1001 int * node_backend_id = &tensor_backend_id(node);1002 if (*node_backend_id == -1) {1003 // unassigned node: find the backend with the most supported inputs1004 int n_supported_best = -1;1005 for (int b = 0; b < sched->n_backends; b++) {1006 if (ggml_backend_supports_op(sched->backends[b], node)) {1007 int n_supported = 0;1008 for (int j = 0; j < GGML_MAX_SRC; j++) {1009 struct ggml_tensor * src = node->src[j];1010 if (src == NULL) {1011 continue;1012 }1013 if ((tensor_backend_id(src) != -1 || tensor_backend_id(src->view_src) != -1) && ggml_backend_sched_buffer_supported(sched, src, b)) {1014 n_supported++;1015 }1016 }1017 if (n_supported > n_supported_best) {1018 n_supported_best = n_supported;1019 *node_backend_id = b;1020 SET_CAUSE(node, "3.best");1021 }1022 }1023 }1024 } else {1025 // assigned node: upgrade to higher prio backend if possible1026 for (int b = 0; b < *node_backend_id; b++) {1027 if (sched->bufts[b] == sched->bufts[*node_backend_id] && ggml_backend_supports_op(sched->backends[b], node)) {1028 bool supported = true;1029 for (int j = 0; j < GGML_MAX_SRC; j++) {1030 struct ggml_tensor * src = node->src[j];1031 if (src == NULL) {1032 continue;1033 }1034 if (!ggml_backend_sched_buffer_supported(sched, src, b)) {1035 supported = false;1036 break;1037 }1038 }1039 if (supported) {1040 *node_backend_id = b;1041 SET_CAUSE(node, "3.upg");1042 break;1043 }1044 }1045 }1046 }1047 }1048 1049 // pass 4: assign backends to remaining src from dst and view_src1050 for (int i = 0; i < graph->n_nodes; i++) {1051 struct ggml_tensor * node = graph->nodes[i];1052 int * cur_backend_id = &tensor_backend_id(node);1053 if (node->view_src != NULL && *cur_backend_id == -1) {1054 *cur_backend_id = tensor_backend_id(node->view_src);1055 SET_CAUSE(node, "4.vsrc");1056 }1057 for (int j = 0; j < GGML_MAX_SRC; j++) {1058 struct ggml_tensor * src = node->src[j];1059 if (src == NULL) {1060 continue;1061 }1062 int * src_backend_id = &tensor_backend_id(src);1063 if (*src_backend_id == -1) {1064 if (src->view_src != NULL) {1065 // views are always on the same backend as the source1066 *src_backend_id = tensor_backend_id(src->view_src);1067 SET_CAUSE(src, "4.vsrc");1068 } else {1069 *src_backend_id = *cur_backend_id;1070 SET_CAUSE(src, "4.cur");1071 }1072 }1073 }1074 // if the node is still unassigned, assign it to the first backend that supports it1075 for (int b = 0; b < sched->n_backends && *cur_backend_id == -1; b++) {1076 ggml_backend_sched_set_if_supported(sched, node, b, cur_backend_id);1077 }1078 GGML_ASSERT(*cur_backend_id != -1);1079 }1080 1081 // pass 5: split graph, find tensors that need to be copied1082 {1083 int i_split = 0;1084 struct ggml_backend_sched_split * split = &sched->splits[0];1085 // find the backend of the first split, skipping view ops1086 int i = 0;1087 for (; i < graph->n_nodes; i++) {1088 struct ggml_tensor * node = graph->nodes[i];1089 if (!ggml_is_view_op(node->op)) {1090 split->backend_id = tensor_backend_id(node);1091 break;1092 }1093 }1094 split->i_start = 0;1095 split->n_inputs = 0;1096 int cur_backend_id = split->backend_id;1097 for (; i < graph->n_nodes; i++) {1098 struct ggml_tensor * node = graph->nodes[i];1099 1100 if (ggml_is_view_op(node->op)) {1101 continue;1102 }1103 1104 const int node_backend_id = tensor_backend_id(node);1105 1106 GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now, this can happen if there is no CPU fallback1107 1108 // check if we should start a new split based on the sources of the current node1109 bool need_new_split = false;1110 if (node_backend_id == cur_backend_id && split->n_inputs > 0) {1111 for (int j = 0; j < GGML_MAX_SRC; j++) {1112 struct ggml_tensor * src = node->src[j];1113 if (src == NULL) {1114 continue;1115 }1116 // check if a weight is on a different and incompatible backend1117 // by starting a new split, the memory of the previously offloaded weights can be reused1118 if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {1119 int src_backend_id = tensor_backend_id(src);1120 if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {1121 need_new_split = true;1122 break;1123 }1124 }1125 // check if the split has too many inputs1126 // FIXME: count the number of inputs instead of only checking when full1127 if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {1128 const size_t id = hash_id(src);1129 int src_backend_id = sched->hv_tensor_backend_ids[id];1130 bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);1131 if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) {1132 need_new_split = true;1133 break;1134 }1135 }1136 }1137 }1138 1139 if (node_backend_id != cur_backend_id || need_new_split) {1140 split->i_end = i;1141 i_split++;1142 if (i_split >= sched->splits_capacity) {1143 sched->splits_capacity *= 2;1144 sched->splits = (ggml_backend_sched_split *)1145 realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));1146 GGML_ASSERT(sched->splits != NULL);1147 }1148 split = &sched->splits[i_split];1149 split->backend_id = node_backend_id;1150 split->i_start = i;1151 split->n_inputs = 0;1152 cur_backend_id = node_backend_id;1153 }1154 1155 // find inputs that are not on the same backend1156 for (int j = 0; j < GGML_MAX_SRC; j++) {1157 struct ggml_tensor * src = node->src[j];1158 if (src == NULL) {1159 continue;1160 }1161 1162 size_t src_id = hash_id(src);1163 const int src_backend_id = sched->hv_tensor_backend_ids[src_id];1164 GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now1165 1166 if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {1167 if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) {1168 ggml_backend_t backend = sched->backends[src_backend_id];1169 for (int c = 0; c < sched->n_copies; c++) {1170 struct ggml_tensor * tensor_copy;1171 if (c == sched->cur_copy) {1172 tensor_copy = src; // use the original tensor as the current copy1173 } else {1174 tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);1175 ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);1176 }1177 if (sched->n_copies > 1) {1178 ggml_set_input(tensor_copy);1179 ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor1180 }1181 tensor_id_copy(src_id, src_backend_id, c) = tensor_copy;1182 SET_CAUSE(tensor_copy, "4.cpy");1183 }1184 int n_graph_inputs = sched->n_graph_inputs++;1185 GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);1186 sched->graph_inputs[n_graph_inputs] = src;1187 }1188 }1189 1190 if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {1191 // create a copy of the input in the split's backend1192 if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) {1193 ggml_backend_t backend = sched->backends[cur_backend_id];1194 for (int c = 0; c < sched->n_copies; c++) {1195 struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);1196 ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);1197 if (sched->n_copies > 1) {1198 ggml_set_input(tensor_copy);1199 ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor1200 }