Brunobkr/llama.cpp_AlgMor24_github

1// Note: porting this file to C++ is a work in progress2 3#ifdef _WIN324#define WIN32_LEAN_AND_MEAN5#ifndef NOMINMAX6#   define NOMINMAX7#endif8#include <windows.h>9#endif10 11#include "ggml-backend.h"12#include "ggml-backend-impl.h"13#include "ggml-alloc.h"14#include "ggml-impl.h"15 16#include <assert.h>17#include <limits.h>18#include <stdarg.h>19#include <stdio.h>20#include <stdlib.h>21#include <string.h>22#include <algorithm>23#include <vector>24 25#ifdef __APPLE__26#include <sys/types.h>27#include <sys/sysctl.h>28#endif29 30 31// backend buffer type32 33const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {34    GGML_ASSERT(buft);35    return buft->iface.get_name(buft);36}37 38ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {39    GGML_ASSERT(buft);40    if (size == 0) {41        // return a dummy buffer for zero-sized allocations42        return ggml_backend_buffer_init(buft, {}, NULL, 0);43    }44    return buft->iface.alloc_buffer(buft, size);45}46 47size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {48    GGML_ASSERT(buft);49    return buft->iface.get_alignment(buft);50}51 52size_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {53    GGML_ASSERT(buft);54    // get_max_size is optional, defaults to SIZE_MAX55    if (buft->iface.get_max_size) {56        return buft->iface.get_max_size(buft);57    }58    return SIZE_MAX;59}60 61size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {62    GGML_ASSERT(buft);63    // get_alloc_size is optional, defaults to ggml_nbytes64    if (buft->iface.get_alloc_size) {65        size_t size = buft->iface.get_alloc_size(buft, tensor);66        assert(size >= ggml_nbytes(tensor));67        return size;68    }69    return ggml_nbytes(tensor);70}71 72bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {73    GGML_ASSERT(buft);74    if (buft->iface.is_host) {75        return buft->iface.is_host(buft);76    }77    return false;78}79 80ggml_backend_dev_t ggml_backend_buft_get_device(ggml_backend_buffer_type_t buft) {81    GGML_ASSERT(buft);82    return buft->device;83}84 85// backend buffer86 87ggml_backend_buffer_t ggml_backend_buffer_init(88               ggml_backend_buffer_type_t buft,89        struct ggml_backend_buffer_i      iface,90               void *                     context,91               size_t                     size) {92    ggml_backend_buffer_t buffer = new ggml_backend_buffer {93        /* .interface = */ iface,94        /* .buft      = */ buft,95        /* .context   = */ context,96        /* .size      = */ size,97        /* .usage     = */ GGML_BACKEND_BUFFER_USAGE_ANY98    };99 100    return buffer;101}102 103const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {104    return ggml_backend_buft_name(ggml_backend_buffer_get_type(buffer));105}106 107void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {108    if (buffer == NULL) {109        return;110    }111 112    if (buffer->iface.free_buffer != NULL) {113        buffer->iface.free_buffer(buffer);114    }115    delete buffer;116}117 118size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {119    GGML_ASSERT(buffer);120    return buffer->size;121}122 123void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {124    GGML_ASSERT(buffer);125    // get_base is optional if the buffer is zero-sized126    if (!ggml_backend_buffer_is_meta(buffer) && buffer->size == 0) {127        return NULL;128    }129 130    // FIXME JG: a multi_buffer has a non-zero size, according to the above comment get_base is not optional,131    //     I don't know whether the above comment is correct132    if (!buffer->iface.get_base) {133        return NULL;134    }135 136    void * base = buffer->iface.get_base(buffer);137 138    GGML_ASSERT(base != NULL && "backend buffer base cannot be NULL");139 140    return base;141}142 143enum ggml_status ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {144    GGML_ASSERT(buffer);145    // init_tensor is optional146    if (buffer->iface.init_tensor) {147        return buffer->iface.init_tensor(buffer, tensor);148    }149    return GGML_STATUS_SUCCESS;150}151 152void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {153    GGML_ASSERT(buffer);154    // clear is optional if the buffer is zero-sized155    if (buffer->size == 0) {156        return;157    }158 159    buffer->iface.clear(buffer, value);160}161 162size_t ggml_backend_buffer_get_alignment(ggml_backend_buffer_t buffer) {163    return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));164}165 166size_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {167    return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));168}169 170size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor) {171    return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);172}173 174bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {175    return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));176}177 178void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {179    GGML_ASSERT(buffer);180    buffer->usage = usage;181 182    // FIXME: add a generic callback to the buffer interface183    if (ggml_backend_buffer_is_multi_buffer(buffer)) {184        ggml_backend_multi_buffer_set_usage(buffer, usage);185    }186}187 188enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t buffer) {189    GGML_ASSERT(buffer);190    return buffer->usage;191}192 193ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {194    GGML_ASSERT(buffer);195    return buffer->buft;196}197 198void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {199    GGML_ASSERT(buffer);200    if (buffer->iface.reset) {201        buffer->iface.reset(buffer);202    }203}204 205bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {206    ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;207    if (dst_buf->iface.cpy_tensor) {208        return dst_buf->iface.cpy_tensor(dst_buf, src, dst);209    }210    return false;211}212 213// backend214 215ggml_guid_t ggml_backend_guid(ggml_backend_t backend) {216    if (backend == NULL) {217        return NULL;218    }219    return backend->guid;220}221 222const char * ggml_backend_name(ggml_backend_t backend) {223    if (backend == NULL) {224        return "NULL";225    }226    return backend->iface.get_name(backend);227}228 229void ggml_backend_free(ggml_backend_t backend) {230    if (backend == NULL) {231        return;232    }233 234    backend->iface.free(backend);235}236 237ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {238    GGML_ASSERT(backend);239    return ggml_backend_dev_buffer_type(backend->device);240}241 242ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {243    return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);244}245 246size_t ggml_backend_get_alignment(ggml_backend_t backend) {247    return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));248}249 250size_t ggml_backend_get_max_size(ggml_backend_t backend) {251    return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));252}253 254void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {255    GGML_ASSERT(backend);256    GGML_ASSERT(tensor);257    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");258    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");259 260    if (backend->iface.set_tensor_async == NULL) {261        ggml_backend_synchronize(backend);262        ggml_backend_tensor_set(tensor, data, offset, size);263    } else {264        backend->iface.set_tensor_async(backend, tensor, data, offset, size);265    }266}267 268void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {269    GGML_ASSERT(backend);270    GGML_ASSERT(tensor);271    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");272    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");273 274    if (backend->iface.get_tensor_async == NULL) {275        ggml_backend_synchronize(backend);276        ggml_backend_tensor_get(tensor, data, offset, size);277    } else {278        backend->iface.get_tensor_async(backend, tensor, data, offset, size);279    }280}281 282void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size,283            size_t n_copies, size_t stride_tensor, size_t stride_data) {284    GGML_ASSERT(backend);285    GGML_ASSERT(tensor);286    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");287 288    if (n_copies <= 1 || backend->iface.set_tensor_2d_async == NULL) {289        for (size_t i = 0; i < n_copies; i++) {290            ggml_backend_tensor_set_async(backend, tensor, (const char *) data + i*stride_data, offset + i*stride_tensor, size);291        }292        return;293    }294    if (size == 0) {295        return;296    }297 298    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");299    GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");300    backend->iface.set_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data);301}302 303void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size,304            size_t n_copies, size_t stride_tensor, size_t stride_data) {305    GGML_ASSERT(backend);306    GGML_ASSERT(tensor);307    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");308 309    if (n_copies <= 1 || backend->iface.get_tensor_2d_async == NULL) {310        for (size_t i = 0; i < n_copies; i++) {311            ggml_backend_tensor_get_async(backend, tensor, (char *) data + i*stride_data, offset + i*stride_tensor, size);312        }313        return;314    }315    if (size == 0) {316        return;317    }318 319    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");320    GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor read out of bounds");321    backend->iface.get_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data);322}323 324void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {325    GGML_ASSERT(tensor);326    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;327    GGML_ASSERT(buf != NULL && "tensor buffer not set");328 329    if (size == 0) {330        return;331    }332 333    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");334    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");335 336    buf->iface.set_tensor(buf, tensor, data, offset, size);337}338 339void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {340    GGML_ASSERT(tensor);341    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;342    GGML_ASSERT(buf != NULL && "tensor buffer not set");343 344    if (size == 0) {345        return;346    }347 348    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");349    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");350 351    buf->iface.get_tensor(buf, tensor, data, offset, size);352}353 354void ggml_backend_tensor_set_2d(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size,355            size_t n_copies, size_t stride_tensor, size_t stride_data) {356    GGML_ASSERT(tensor);357    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;358    GGML_ASSERT(buf != NULL && "tensor buffer not set");359 360    if (n_copies <= 1 || buf->iface.set_tensor_2d == NULL) {361        for (size_t i = 0; i < n_copies; i++) {362            ggml_backend_tensor_set(tensor, (const char *) data + i*stride_data, offset + i*stride_tensor, size);363        }364        return;365    }366    if (size == 0) {367        return;368    }369 370    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");371    GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");372 373    buf->iface.set_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);374}375 376void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size,377            size_t n_copies, size_t stride_tensor, size_t stride_data) {378    GGML_ASSERT(tensor);379    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;380    GGML_ASSERT(buf != NULL && "tensor buffer not set");381 382    if (n_copies <= 1 || buf->iface.get_tensor_2d == NULL) {383        for (size_t i = 0; i < n_copies; i++) {384            ggml_backend_tensor_get(tensor, (char *) data + i*stride_data, offset + i*stride_tensor, size);385        }386        return;387    }388    if (size == 0) {389        return;390    }391 392    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");393    GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor read out of bounds");394 395    buf->iface.get_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);396}397 398void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {399    GGML_ASSERT(tensor);400    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;401 402    if (size == 0) {403        return;404    }405 406    GGML_ASSERT(buf != NULL && "tensor buffer not set");407    GGML_ASSERT(tensor->data != NULL && "tensor not allocated");408    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");409    GGML_ASSERT(buf->iface.memset_tensor != NULL && "memset not implemented by backend buffer");410 411    buf->iface.memset_tensor(buf, tensor, value, offset, size);412}413 414void ggml_backend_synchronize(ggml_backend_t backend) {415    GGML_ASSERT(backend);416    if (backend->iface.synchronize == NULL) {417        return;418    }419 420    backend->iface.synchronize(backend);421}422 423ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {424    GGML_ASSERT(backend);425    GGML_ASSERT(backend->iface.graph_plan_create != NULL);426 427    return backend->iface.graph_plan_create(backend, cgraph);428}429 430void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {431    GGML_ASSERT(backend);432    GGML_ASSERT(backend->iface.graph_plan_free != NULL);433 434    backend->iface.graph_plan_free(backend, plan);435}436 437enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {438    GGML_ASSERT(backend);439    GGML_ASSERT(backend->iface.graph_plan_compute != NULL);440 441    return backend->iface.graph_plan_compute(backend, plan);442}443 444enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {445    enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);446    ggml_backend_synchronize(backend);447    return err;448}449 450enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {451    GGML_ASSERT(backend);452    return backend->iface.graph_compute(backend, cgraph);453}454 455bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {456    GGML_ASSERT(backend);457    return ggml_backend_dev_supports_op(backend->device, op);458}459 460bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {461    GGML_ASSERT(backend);462    return ggml_backend_dev_supports_buft(backend->device, buft);463}464 465bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {466    GGML_ASSERT(backend);467    return ggml_backend_dev_offload_op(backend->device, op);468}469 470ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend) {471    GGML_ASSERT(backend);472    return backend->device;473}474 475// backend copy476 477void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst) {478    GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");479 480    if (src == dst) {481        return;482    }483 484    if (ggml_backend_buffer_is_host(src->buffer)) {485        ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));486    } else if (ggml_backend_buffer_is_host(dst->buffer)) {487        ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));488    } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {489#ifndef NDEBUG490        GGML_LOG_DEBUG("%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));491#endif // NDEBUG492        size_t nbytes = ggml_nbytes(src);493        void * data = malloc(nbytes);494        ggml_backend_tensor_get(src, data, 0, nbytes);495        ggml_backend_tensor_set(dst, data, 0, nbytes);496        free(data);497    }498}499 500void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {501    GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");502 503    if (src == dst) {504        return;505    }506 507    GGML_ASSERT(backend_dst);508    if (backend_dst->iface.cpy_tensor_async != NULL) {509        if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {510            return;511        }512    }513 514    // an async copy would normally happen after all the queued operations on both backends are completed515    // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy516    ggml_backend_synchronize(backend_src);517    ggml_backend_synchronize(backend_dst);518    ggml_backend_tensor_copy(src, dst);519}520 521// events522 523ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device) {524    // null device is allowed for the transition period to the device interface525    if (device == NULL || device->iface.event_new == NULL) {526        return NULL;527    }528    return device->iface.event_new(device);529}530 531void ggml_backend_event_free(ggml_backend_event_t event) {532    if (event == NULL) {533        return;534    }535    event->device->iface.event_free(event->device, event);536}537 538void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend) {539    GGML_ASSERT(backend);540    GGML_ASSERT(backend->iface.event_record != NULL);541 542    backend->iface.event_record(backend, event);543}544 545void ggml_backend_event_synchronize(ggml_backend_event_t event) {546    GGML_ASSERT(event);547    GGML_ASSERT(event->device->iface.event_synchronize);548 549    event->device->iface.event_synchronize(event->device, event);550}551 552void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {553    GGML_ASSERT(backend);554    GGML_ASSERT(backend->iface.event_wait != NULL);555 556    backend->iface.event_wait(backend, event);557}558 559static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph) {560    GGML_ASSERT(backend);561    if (backend->iface.graph_optimize != NULL) {562        backend->iface.graph_optimize(backend, cgraph);563    }564}565 566// Backend device567 568const char * ggml_backend_dev_name(ggml_backend_dev_t device) {569    GGML_ASSERT(device);570    return device->iface.get_name(device);571}572 573const char * ggml_backend_dev_description(ggml_backend_dev_t device) {574    GGML_ASSERT(device);575    return device->iface.get_description(device);576}577 578void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total) {579    GGML_ASSERT(device);580    device->iface.get_memory(device, free, total);581}582 583enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device) {584    GGML_ASSERT(device);585    return device->iface.get_type(device);586}587 588void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props) {589    GGML_ASSERT(device);590    memset(props, 0, sizeof(*props));591    device->iface.get_props(device, props);592}593 594ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device) {595    GGML_ASSERT(device);596    return device->reg;597}598 599ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params) {600    GGML_ASSERT(device);601    return device->iface.init_backend(device, params);602}603 604ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device) {605    GGML_ASSERT(device);606    return device->iface.get_buffer_type(device);607}608 609ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device) {610    GGML_ASSERT(device);611    if (device->iface.get_host_buffer_type == NULL) {612        return NULL;613    }614 615    return device->iface.get_host_buffer_type(device);616}617 618ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size) {619    GGML_ASSERT(device);620    return device->iface.buffer_from_host_ptr(device, ptr, size, max_tensor_size);621}622 623bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {624    GGML_ASSERT(device);625    return device->iface.supports_op(device, op);626}627 628bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {629    GGML_ASSERT(device);630    return device->iface.supports_buft(device, buft);631}632 633bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {634    GGML_ASSERT(device);635    if (device->iface.offload_op != NULL) {636        return device->iface.offload_op(device, op);637    }638 639    return false;640}641 642// Backend (reg)643 644const char * ggml_backend_reg_name(ggml_backend_reg_t reg) {645    GGML_ASSERT(reg);646    return reg->iface.get_name(reg);647}648 649size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg) {650    GGML_ASSERT(reg);651    return reg->iface.get_device_count(reg);652}653 654ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index) {655    GGML_ASSERT(reg);656    return reg->iface.get_device(reg, index);657}658 659void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {660    GGML_ASSERT(reg);661    if (!reg->iface.get_proc_address) {662        return NULL;663    }664    return reg->iface.get_proc_address(reg, name);665}666 667// multi-buffer buffer668 669struct ggml_backend_multi_buffer_context {670    ggml_backend_buffer_t * buffers;671    size_t n_buffers;672};673 674static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {675    GGML_ASSERT(buffer);676    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;677    for (size_t i = 0; i < ctx->n_buffers; i++) {678        ggml_backend_buffer_free(ctx->buffers[i]);679    }680 681    free(ctx->buffers);682    free(ctx);683}684 685static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {686    GGML_ASSERT(buffer);687    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;688    for (size_t i = 0; i < ctx->n_buffers; i++) {689        ggml_backend_buffer_clear(ctx->buffers[i], value);690    }691}692 693static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {694    /* .free_buffer     = */ ggml_backend_multi_buffer_free_buffer,695    /* .get_base        = */ NULL,696    /* .init_tensor     = */ NULL,697    /* .memset_tensor   = */ NULL,698    /* .set_tensor      = */ NULL,699    /* .get_tensor      = */ NULL,700    /* .set_tensor_2d   = */ NULL,701    /* .get_tensor_2d   = */ NULL,702    /* .cpy_tensor      = */ NULL,703    /* .clear           = */ ggml_backend_multi_buffer_clear,704    /* .reset           = */ NULL,705};706 707ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {708    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) malloc(sizeof(struct ggml_backend_multi_buffer_context));709    ctx->n_buffers = n_buffers;710    ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));711 712    GGML_ASSERT(ctx->buffers != NULL);713 714    size_t total_size = 0;715    for (size_t i = 0; i < n_buffers; i++) {716        ctx->buffers[i] = buffers[i];717        total_size += ggml_backend_buffer_get_size(buffers[i]);718    }719 720    return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_i, ctx, total_size);721}722 723bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {724    GGML_ASSERT(buffer);725    return buffer->iface.free_buffer == ggml_backend_multi_buffer_free_buffer;726}727 728void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {729    GGML_ASSERT(buffer);730    GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));731    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;732    for (size_t i = 0; i < ctx->n_buffers; i++) {733        ggml_backend_buffer_set_usage(ctx->buffers[i], usage);734    }735}736 737// creates a copy of the tensor with the same memory layout738static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {739    struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);740    for (int i = 0; i < GGML_MAX_DIMS; i++) {741        dup->nb[i] = tensor->nb[i];742    }743    return dup;744}745 746static bool ggml_is_view_op(enum ggml_op op) {747    return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;748}749 750// scheduler751 752#ifndef GGML_SCHED_MAX_BACKENDS753#define GGML_SCHED_MAX_BACKENDS 16754#endif755 756#ifndef GGML_SCHED_MAX_SPLIT_INPUTS757#define GGML_SCHED_MAX_SPLIT_INPUTS 30758#endif759 760#ifndef GGML_SCHED_MAX_COPIES761#define GGML_SCHED_MAX_COPIES 4762#endif763 764struct ggml_backend_sched_split {765    int backend_id;766    int i_start;767    int i_end;768    struct ggml_tensor ** inputs;769    int n_inputs;770    int inputs_capacity;771    // graph view of this split772    struct ggml_cgraph graph;773};774 775struct ggml_backend_sched {776    bool is_reset; // true if the scheduler has been reset since the last graph split777    bool is_alloc;778 779    int n_backends;780 781    ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];782    ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];783    ggml_gallocr_t galloc;784 785    // hash map of the nodes in the graph786    struct ggml_hash_set  hash_set;787    int                 * hv_tensor_backend_ids; // [hash_set.size]788    struct ggml_tensor ** hv_tensor_copies;      // [hash_set.size][n_backends][n_copies]789 790    int * node_backend_ids; // [graph_size]791    int * leaf_backend_ids; // [graph_size]792 793    int * prev_node_backend_ids; // [graph_size]794    int * prev_leaf_backend_ids; // [graph_size]795 796    // copy of the graph with modified inputs797    struct ggml_cgraph graph;798 799    // graph splits800    struct ggml_backend_sched_split * splits;801    int n_splits;802    int splits_capacity;803 804    // pipeline parallelism support805    int n_copies;806    int cur_copy;807    int next_copy;808    ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];809    struct ggml_tensor ** graph_inputs;810    int n_graph_inputs;811    int graph_inputs_capacity;812 813    struct ggml_context * ctx;814 815    ggml_backend_sched_eval_callback callback_eval;816    void * callback_eval_user_data;817 818    char * context_buffer;819    size_t context_buffer_size;820 821    bool op_offload;822 823    int debug;824 825    // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC]826    // ref: https://github.com/ggml-org/llama.cpp/pull/17617827    int debug_realloc;828    int debug_graph_size;829    int debug_prev_graph_size;830};831 832#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)833#define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]834#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]835#define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)836 837static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split * split) {838    int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;839    if (split->inputs_capacity > 0) {840        new_cap = 2*split->inputs_capacity;841        GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap);842    }843    auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *));844    if (pnew == NULL) {845        GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));846        GGML_ABORT("failed to grow split inputs container");847    }848    split->inputs = pnew;849    split->inputs_capacity = new_cap;850}851 852static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) {853    int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;854    if (sched->graph_inputs_capacity > 0) {855        new_cap = 2*sched->graph_inputs_capacity;856        GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap);857    }858    auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *));859    if (pnew == NULL) {860        GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));861        GGML_ABORT("failed to grow graph inputs container");862    }863    sched->graph_inputs = pnew;864    sched->graph_inputs_capacity = new_cap;865}866 867// returns the priority of the backend, lower id is higher priority868static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {869    for (int i = 0; i < sched->n_backends; i++) {870        if (sched->backends[i] == backend) {871            return i;872        }873    }874    return -1;875}876 877static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor, const struct ggml_tensor * op) {878    ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;879    if (buffer == NULL) {880        return -1;881    }882 883    // find highest prio backend that supports the buffer type and the op884    for (int i = 0; i < sched->n_backends; i++) {885        if (ggml_backend_supports_buft(sched->backends[i], buffer->buft) &&886            ggml_backend_supports_op(sched->backends[i], op)) {887            return i;888        }889    }890 891#ifndef NDEBUG892    GGML_LOG_DEBUG("%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\n",893        __func__, ggml_op_desc(tensor), ggml_backend_buffer_name(buffer), tensor->name);894#endif895 896    return -1;897}898 899#if 0900#define GGML_SCHED_MAX_SPLITS_DEBUG 4096901static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only902#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)903#define GET_CAUSE(node) causes[hash_id(node)]904#else905#define SET_CAUSE(node, ...)906#define GET_CAUSE(node) ""907#endif908 909// returns the backend that should be used for the node based on the current locations910static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {911    // assign pre-allocated nodes to their backend912    int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);913    if (cur_backend_id != -1) {914        SET_CAUSE(tensor, "1.dst");915        return cur_backend_id;916    }917 918    // view_src919    if (tensor->view_src != NULL) {920        cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src, tensor);921        if (cur_backend_id != -1) {922            SET_CAUSE(tensor, "1.vsrc");923            return cur_backend_id;924        }925    }926 927    if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {928        // since the tensor is pre-allocated, it cannot be moved to another backend929        ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;930        GGML_ABORT("pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op));931    }932 933    // graph input934    if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {935        cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)936        SET_CAUSE(tensor, "1.inp");937        return cur_backend_id;938    }939 940    // operations with weights are preferably run on the same backend as the weights941    // TODO: there are exceptions (see below) - not an ideal solution942    bool allow = true;943 944    // skip ROPE since the rope freqs tensor is too small to choose a backend based on it945    allow = allow && tensor->op != GGML_OP_ROPE;946 947    // skip FLASH_ATTN_EXT since the sinks tensor is too small to choose a based based on it948    allow = allow && tensor->op != GGML_OP_FLASH_ATTN_EXT;949 950    if (allow) {951        for (int i = 0; i < GGML_MAX_SRC; i++) {952            const struct ggml_tensor * src = tensor->src[i];953            if (src == NULL) {954                continue;955            }956            if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {957                int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor);958                // check if a backend with higher prio wants to offload the op959                if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) {960                    for (int b = 0; b < src_backend_id; b++) {961                        if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) {962                            SET_CAUSE(tensor, "1.off");963                            return b;964                        }965                    }966                }967                SET_CAUSE(tensor, "1.wgt%d", i);968                return src_backend_id;969            }970        }971    }972 973    return -1;974}975 976static char * fmt_size(size_t size) {977    static char buffer[128];978    if (size >= 1024*1024) {979        snprintf(buffer, sizeof(buffer), "%zuM", size/1024/1024);980    } else {981        snprintf(buffer, sizeof(buffer), "%zuK", size/1024);982    }983    return buffer;984}985 986static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {987    int cur_split = 0;988    for (int i = 0; i < graph->n_nodes; i++) {989        if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {990            ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];991            GGML_LOG_DEBUG("\n## SPLIT #%d: %s # %d inputs", cur_split, ggml_backend_name(split_backend),992                sched->splits[cur_split].n_inputs);993            for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {994                if (j == 0) {995                    GGML_LOG_DEBUG(": ");996                }997                GGML_LOG_DEBUG("[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,998                    fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));999            }1000            GGML_LOG_DEBUG("\n");1001            cur_split++;1002        }1003        struct ggml_tensor * node = graph->nodes[i];1004        if (ggml_is_view_op(node->op)) {1005            continue;1006        }1007        if (sched->debug > 1) {1008            ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);1009            GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_desc(node), node->name,1010                fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node),1011                graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0);1012            for (int j = 0; j < GGML_MAX_SRC; j++) {1013                struct ggml_tensor * src = node->src[j];1014                if (src == NULL) {1015                    continue;1016                }1017                ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);1018                GGML_LOG_DEBUG(" %20.20s (%5.5s) [%5.5s %8.8s]", src->name,1019                    fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));1020            }1021            GGML_LOG_DEBUG("\n");1022        }1023    }1024}1025 1026static bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) {1027    ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer;1028    ggml_backend_buffer_type_t buft = NULL;1029 1030    if (buf) {1031        // the tensor is already allocated1032        buft = buf->buft;1033    } else {1034        // see if the tensor already has a backend assigned, and use the buffer type of that backend1035        int tensor_backend_id = tensor_backend_id(t);1036        if (tensor_backend_id == -1 && t->view_src) {1037            tensor_backend_id = tensor_backend_id(t->view_src);1038        }1039        if (tensor_backend_id != -1) {1040            buft = sched->bufts[tensor_backend_id];1041        }1042    }1043 1044    return buft != NULL && ggml_backend_supports_buft(sched->backends[backend_id], buft);1045}1046 1047static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, struct ggml_tensor * node, int cur_backend_id, int * node_backend_id) {1048    if (ggml_backend_supports_op(sched->backends[cur_backend_id], node)) {1049        *node_backend_id = cur_backend_id;1050        SET_CAUSE(node, "2.sup");1051    }1052}1053 1054// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend1055void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {1056    // reset splits1057    sched->n_splits = 0;1058    sched->n_graph_inputs = 0;1059    sched->is_reset = false;1060 1061    struct ggml_init_params params = {1062        /* .mem_size =   */ sched->context_buffer_size,1063        /* .mem_buffer = */ sched->context_buffer,1064        /* .no_alloc =   */ true1065    };1066 1067    ggml_free(sched->ctx);1068 1069    sched->ctx = ggml_init(params);1070    if (sched->ctx == NULL) {1071        GGML_ABORT("%s: failed to initialize context\n", __func__);1072    }1073 1074    graph->uid = ggml_graph_next_uid();1075 1076    // pass 1: assign backends to ops with pre-allocated inputs1077    for (int i = 0; i < graph->n_leafs; i++) {1078        struct ggml_tensor * leaf = graph->leafs[i];1079        int * leaf_backend_id = &tensor_backend_id(leaf);1080        // do not overwrite user assignments1081        if (*leaf_backend_id == -1) {1082            *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);1083        }1084    }1085 1086    for (int i = 0; i < graph->n_nodes; i++) {1087        struct ggml_tensor * node = graph->nodes[i];1088        int * node_backend_id = &tensor_backend_id(node);1089        // do not overwrite user assignments1090        if (*node_backend_id == -1) {1091            *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);1092 1093#if 01094            // src1095            if (node->op == GGML_OP_NONE) {1096                continue;1097            }1098 1099            for (int j = 0; j < GGML_MAX_SRC; j++) {1100                struct ggml_tensor * src = node->src[j];1101                if (src == NULL) {1102                    continue;1103                }1104                int * src_backend_id = &tensor_backend_id(src);1105                if (*src_backend_id == -1) {1106                    *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);1107                }1108            }1109#endif1110        }1111    }1112 1113    // pass 2: expand current backend assignments1114    // assign the same backend to adjacent nodes1115    // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)1116    // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops1117    // ops unsupported by the backend being expanded will be left unassigned so that they can be assigned later when the locations of its inputs are known1118    // expand gpu down1119    {1120        int cur_backend_id = -1;1121        for (int i = 0; i < graph->n_nodes; i++) {1122            struct ggml_tensor * node = graph->nodes[i];1123            if (ggml_is_view_op(node->op)) {1124                continue;1125            }1126            int * node_backend_id = &tensor_backend_id(node);1127            if (*node_backend_id != -1) {1128                if (*node_backend_id == sched->n_backends - 1) {1129                    // skip cpu (lowest prio backend)1130                    cur_backend_id = -1;1131                } else {1132                    cur_backend_id = *node_backend_id;1133                }1134            } else if (cur_backend_id != -1) {1135                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);1136            }1137        }1138    }1139    // expand gpu up1140    {1141        int cur_backend_id = -1;1142        for (int i = graph->n_nodes - 1; i >= 0; i--) {1143            struct ggml_tensor * node = graph->nodes[i];1144            if (ggml_is_view_op(node->op)) {1145                continue;1146            }1147            int * node_backend_id = &tensor_backend_id(node);1148            if (*node_backend_id != -1) {1149                if (*node_backend_id == sched->n_backends - 1) {1150                    // skip cpu (lowest prio backend)1151                    cur_backend_id = -1;1152                } else {1153                    cur_backend_id = *node_backend_id;1154                }1155            } else if (cur_backend_id != -1) {1156                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);1157            }1158        }1159    }1160    // expand rest down1161    {1162        int cur_backend_id = -1;1163        for (int i = 0; i < graph->n_nodes; i++) {1164            struct ggml_tensor * node = graph->nodes[i];1165            if (ggml_is_view_op(node->op)) {1166                continue;1167            }1168            int * node_backend_id = &tensor_backend_id(node);1169            if (*node_backend_id != -1) {1170                cur_backend_id = *node_backend_id;1171            } else if (cur_backend_id != -1) {1172                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);1173            }1174        }1175    }1176    // expand rest up1177    {1178        int cur_backend_id = -1;1179        for (int i = graph->n_nodes - 1; i >= 0; i--) {1180            struct ggml_tensor * node = graph->nodes[i];1181            if (ggml_is_view_op(node->op)) {1182                continue;1183            }1184            int * node_backend_id = &tensor_backend_id(node);1185            if (*node_backend_id != -1) {1186                cur_backend_id = *node_backend_id;1187            } else if (cur_backend_id != -1) {1188                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);1189            }1190        }1191    }1192 1193    // pass 3: upgrade nodes to higher prio backends with compatible buffer types1194    // if the tensor is already in the same buffer type (*) as another higher priority backend, we should move it there1195    // however, we also need to verify that the sources are in compatible buffer types1196    // (*) the actual requirement is more relaxed, the buffer type of the backend should be supported by all the users of this tensor further down the graph1197    // however, this is slow to verify, so we have a more strict requirement that the buffer type is the same1198    // this is not uncommon since multiple backends can use host memory, with the same buffer type (eg. BLAS and CPU)1199    // additionally, set remaining unassigned nodes to the backend with the most supported inputs1200    // only nodes that could not be assigned during expansion due to the backend not supporting the op should be unassigned at this point

Showing the first 1,200 of 2432 lines. Download the file for the rest.