KBaba7/llama.cpp
0
1#pragma once2 3#include "ggml.h"4#include "ggml-alloc.h"5 6#ifdef GGML_BACKEND_SHARED7# if defined(_WIN32) && !defined(__MINGW32__)8# ifdef GGML_BACKEND_BUILD9# define GGML_BACKEND_API __declspec(dllexport) extern10# else11# define GGML_BACKEND_API __declspec(dllimport) extern12# endif13# else14# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern15# endif16#else17# define GGML_BACKEND_API extern18#endif19 20#ifdef __cplusplus21extern "C" {22#endif23 24 typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;25 typedef struct ggml_backend_buffer * ggml_backend_buffer_t;26 typedef struct ggml_backend_event * ggml_backend_event_t;27 typedef struct ggml_backend * ggml_backend_t;28 typedef void * ggml_backend_graph_plan_t;29 typedef struct ggml_backend_reg * ggml_backend_reg_t;30 typedef struct ggml_backend_device * ggml_backend_dev_t;31 32 33 //34 // Backend buffer type35 //36 37 GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);38 GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);39 GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);40 GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);41 GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);42 GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);43 GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft);44 45 //46 // Backend buffer47 //48 49 enum ggml_backend_buffer_usage {50 GGML_BACKEND_BUFFER_USAGE_ANY = 0,51 GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,52 GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,53 };54 55 GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);56 GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);57 GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);58 GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);59 GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);60 GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);61 GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);62 GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);63 GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);64 GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);65 GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);66 GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);67 GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);68 GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);69 70 // tensor copy between different backends71 GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);72 73 //74 // Backend (stream)75 //76 77 GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);78 GGML_API const char * ggml_backend_name(ggml_backend_t backend);79 GGML_API void ggml_backend_free(ggml_backend_t backend);80 81 GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);82 GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);83 GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);84 GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);85 86 GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);87 GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);88 89 // "offset" refers to the offset in tensor->data for setting/getting data90 GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);91 GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);92 GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);93 94 GGML_API void ggml_backend_synchronize(ggml_backend_t backend);95 96 GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);97 GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);98 99 GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);100 GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);101 GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);102 103 // NOTE: will be removed, use device version instead104 GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);105 GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);106 GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);107 108 // asynchronous copy109 // the copy is performed after all the currently queued operations in backend_src110 // backend_dst will wait for the copy to complete before performing other operations111 // automatic fallback to sync copy if async is not supported112 GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);113 114 GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);115 116 //117 // Events118 //119 120 GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);121 GGML_API void ggml_backend_event_free(ggml_backend_event_t event);122 GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);123 GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);124 GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);125 126 //127 // Backend device128 //129 130 enum ggml_backend_dev_type {131 // CPU device using system memory132 GGML_BACKEND_DEVICE_TYPE_CPU,133 // GPU device using dedicated memory134 GGML_BACKEND_DEVICE_TYPE_GPU,135 // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)136 GGML_BACKEND_DEVICE_TYPE_ACCEL137 };138 139 // functionality supported by the device140 struct ggml_backend_dev_caps {141 // asynchronous operations142 bool async;143 // pinned host buffer144 bool host_buffer;145 // creating buffers from host ptr146 bool buffer_from_host_ptr;147 // event synchronization148 bool events;149 };150 151 // all the device properties152 struct ggml_backend_dev_props {153 const char * name;154 const char * description;155 size_t memory_free;156 size_t memory_total;157 enum ggml_backend_dev_type type;158 struct ggml_backend_dev_caps caps;159 };160 161 GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);162 GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device);163 GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);164 GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device);165 GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);166 GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device);167 GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);168 GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device);169 GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);170 GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);171 172 GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);173 GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);174 GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);175 176 //177 // Backend (reg)178 //179 180 GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg);181 GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg);182 GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);183 GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);184 185 // Common functions that may be obtained using ggml_backend_reg_get_proc_address186 187 // Split buffer type for tensor parallelism188 typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);189 // Set the number of threads for the backend190 typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);191 // Get additional buffer types provided by the device (returns a NULL-terminated array)192 typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);193 // Set the abort callback for the backend194 typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);195 // Get a list of feature flags supported by the backend (returns a NULL-terminated array)196 struct ggml_backend_feature {197 const char * name;198 const char * value;199 };200 typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);201 202 //203 // Backend registry204 //205 206 GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);207 208 // Backend (reg) enumeration209 GGML_API size_t ggml_backend_reg_count(void);210 GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);211 GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);212 213 // Device enumeration214 GGML_API size_t ggml_backend_dev_count(void);215 GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);216 GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);217 GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);218 219 // Direct backend (stream) initialization220 // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)221 GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);222 // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)223 GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);224 // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)225 GGML_API ggml_backend_t ggml_backend_init_best(void);226 227 // Load a backend from a dynamic library and register it228 GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);229 // Unload a backend if loaded dynamically and unregister it230 GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);231 // Load all known backends from dynamic libraries232 GGML_API void ggml_backend_load_all(void);233 GGML_API void ggml_backend_load_all_from_path(const char * dir_path);234 235 //236 // Backend scheduler237 //238 239 // The backend scheduler allows for multiple backend devices to be used together240 // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends241 // The backends are selected based on:242 // - the backend that supports the operation243 // - the location of the pre-allocated tensors (e.g. the weights)244 /*245 Example usage:246 247 // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned248 // preferrably to run on the same backend as the buffer249 ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);250 251 sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);252 253 // initialize buffers from a max size graph (optional)254 reserve_graph = build_graph(sched, max_batch_size);255 256 // manually assign nodes to a backend (optional, should not be needed in most cases)257 struct ggml_tensor * node = ggml_mul_mat(ctx, ...);258 ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);259 260 ggml_backend_sched_reserve(sched, reserve_graph);261 262 // compute263 graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation264 for (int i = 0; i < 10; ++i) {265 ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically266 }267 268 // if there are graph inputs:269 graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)270 ggml_backend_sched_reset(sched); // clear the allocation of the previous graph271 ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it272 ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors273 ggml_backend_sched_graph_compute(sched, graph); // execute the graph274 275 // as an alternative to the above it is also possible to assign the inputs to a dedicated context and276 // allocate them statically via ggml_backend_alloc_ctx_tensors277 }278 */279 280 typedef struct ggml_backend_sched * ggml_backend_sched_t;281 282 // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)283 // when ask == true, the scheduler wants to know if the user wants to observe this node284 // this allows the scheduler to batch nodes together in order to evaluate them in a single call285 //286 // when ask == false, the scheduler is passing the node tensor to the user for observation287 // if the user returns false, the scheduler will cancel the graph compute288 //289 typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);290 291 // Initialize a backend scheduler, backends with low index are given priority over backends with high index292 GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);293 GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);294 295 // Initialize backend buffers from a measure graph296 GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success297 298 GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);299 GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);300 301 // Get the number of splits of the last graph302 GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);303 GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);304 305 GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);306 307 GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);308 GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);309 310 // Allocate and compute graph on the backend scheduler311 GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success312 GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);313 GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);314 GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);315 316 // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.317 // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.318 // The correct way to use this API is to discard the deallocated tensors and create new ones.319 GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);320 321 // Set a callback to be called for each resulting node during graph compute322 GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);323 324 //325 // Utils326 //327 328 struct ggml_backend_graph_copy {329 ggml_backend_buffer_t buffer;330 struct ggml_context * ctx_allocated;331 struct ggml_context * ctx_unallocated;332 struct ggml_cgraph * graph;333 };334 335 // Copy a graph to a different backend336 GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);337 GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);338 339 typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);340 341 // Compare the output of two backends342 GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);343 344 // Tensor initialization345 GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);346 GGML_API void ggml_backend_view_init(struct ggml_tensor * tensor);347 348 // CPU buffer types are always available349 GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);350 GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);351 352#ifdef __cplusplus353}354#endif355