KBaba7/llama.cpp
0
1#include "ggml-vulkan.h"2#include <vulkan/vulkan_core.h>3#if defined(GGML_VULKAN_RUN_TESTS) || defined(GGML_VULKAN_PERF) || defined(GGML_VULKAN_CHECK_RESULTS)4#include <chrono>5#include "ggml-cpu.h"6#endif7 8#include <vulkan/vulkan.hpp>9 10#include <algorithm>11#include <cmath>12#include <iomanip>13#include <iostream>14#include <tuple>15#include <vector>16#include <sstream>17#include <utility>18#include <memory>19#include <limits>20#include <map>21#include <unordered_map>22#include <memory>23#include <mutex>24#include <future>25#include <thread>26 27#include "ggml-impl.h"28#include "ggml-backend-impl.h"29 30#include "ggml-vulkan-shaders.hpp"31 32#define CEIL_DIV(M, N) (((M) + (N)-1) / (N))33 34#define VK_VENDOR_ID_AMD 0x100235#define VK_VENDOR_ID_APPLE 0x106b36#define VK_VENDOR_ID_INTEL 0x808637#define VK_VENDOR_ID_NVIDIA 0x10de38 39#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 3240 41#define GGML_VK_MAX_NODES 819242 43#define MAX_VK_BUFFERS 25644 45#define VK_CHECK(err, msg) \46 do { \47 vk::Result err_ = (err); \48 if (err_ != vk::Result::eSuccess) { \49 fprintf(stderr, "ggml_vulkan: %s error %s at %s:%d\n", \50 #err, to_string(err_).c_str(), __FILE__, __LINE__); \51 exit(1); \52 } \53 } while (0)54 55#ifdef GGML_VULKAN_DEBUG56#define VK_LOG_DEBUG(msg) std::cerr << msg << std::endl57#else58#define VK_LOG_DEBUG(msg) ((void) 0)59#endif // GGML_VULKAN_DEBUG60 61struct ggml_backend_vk_context;62 63struct vk_queue {64 uint32_t queue_family_index;65 vk::Queue queue;66 vk::CommandPool pool;67 uint32_t cmd_buffer_idx;68 std::vector<vk::CommandBuffer> cmd_buffers;69 70 vk::PipelineStageFlags stage_flags;71 72 bool transfer_only;73};74 75struct vk_pipeline_struct {76 std::string name;77 vk::ShaderModule shader_module;78 vk::DescriptorSetLayout dsl;79 std::vector<vk::DescriptorPool> descriptor_pools;80 std::vector<vk::DescriptorSet> descriptor_sets;81 uint32_t descriptor_set_idx;82 vk::PipelineLayout layout;83 vk::Pipeline pipeline;84 uint32_t push_constant_size;85 uint32_t parameter_count;86 std::array<uint32_t, 3> wg_denoms;87 uint32_t align;88 // set to true to request the pipeline is compiled after the dryrun89 bool needed {};90 // set to true when the shader has been compiled91 bool compiled {};92};93 94typedef std::shared_ptr<vk_pipeline_struct> vk_pipeline;95typedef std::weak_ptr<vk_pipeline_struct> vk_pipeline_ref;96 97static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline);98 99struct vk_matmul_pipeline_struct {100 vk_pipeline l, m, s;101 vk_pipeline a_l, a_m, a_s;102};103 104typedef std::shared_ptr<vk_matmul_pipeline_struct> vk_matmul_pipeline;105 106struct vk_matmul_pipeline2 {107 vk_matmul_pipeline2() {108 f16acc = std::make_shared<vk_matmul_pipeline_struct>();109 f32acc = std::make_shared<vk_matmul_pipeline_struct>();110 }111 vk_matmul_pipeline f32acc;112 vk_matmul_pipeline f16acc;113};114 115struct vk_device_struct;116typedef std::shared_ptr<vk_device_struct> vk_device;117typedef std::weak_ptr<vk_device_struct> vk_device_ref;118 119struct vk_buffer_struct;120typedef std::shared_ptr<vk_buffer_struct> vk_buffer;121typedef std::weak_ptr<vk_buffer_struct> vk_buffer_ref;122 123struct ggml_backend_vk_buffer_type_context {124 std::string name;125 vk_device device;126};127 128static const char * ggml_backend_vk_buffer_type_name(ggml_backend_buffer_type_t buft);129static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);130static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft);131static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);132static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);133static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {134 /* .get_name = */ ggml_backend_vk_buffer_type_name,135 /* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,136 /* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,137 /* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,138 /* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,139 /* .is_host = */ NULL,140};141 142#ifdef GGML_VULKAN_MEMORY_DEBUG143class vk_memory_logger;144#endif145#ifdef GGML_VULKAN_PERF146class vk_perf_logger;147#endif148static void ggml_vk_destroy_buffer(vk_buffer& buf);149 150static constexpr uint32_t mul_mat_vec_max_cols = 8;151 152struct vk_device_struct {153 std::mutex mutex;154 155 vk::PhysicalDevice physical_device;156 vk::PhysicalDeviceProperties properties;157 std::string name;158 uint64_t max_memory_allocation_size;159 uint64_t suballocation_block_size;160 bool fp16;161 bool pipeline_robustness;162 vk::Device device;163 uint32_t vendor_id;164 vk_queue compute_queue;165 vk_queue transfer_queue;166 bool single_queue;167 uint32_t subgroup_size;168 uint32_t shader_core_count;169 bool uma;170 bool float_controls_rte_fp16;171 172 bool subgroup_size_control;173 uint32_t subgroup_min_size;174 uint32_t subgroup_max_size;175 bool subgroup_require_full_support;176 177 bool coopmat_support;178 bool coopmat_acc_f32_support;179 bool coopmat_acc_f16_support;180 uint32_t coopmat_m;181 uint32_t coopmat_n;182 uint32_t coopmat_k;183 bool coopmat2;184 185 size_t idx;186 187 bool mul_mat_l;188 bool mul_mat_m;189 bool mul_mat_s;190 bool mul_mat_id_l;191 bool mul_mat_id_m;192 bool mul_mat_id_s;193 194 // set to true to indicate that some shaders need to be compiled after the dryrun195 bool need_compiles {};196 197 vk_matmul_pipeline pipeline_matmul_f32 {};198 vk_matmul_pipeline pipeline_matmul_f32_f16 {};199 vk_matmul_pipeline2 pipeline_matmul_f16;200 vk_matmul_pipeline2 pipeline_matmul_f16_f32;201 vk_pipeline pipeline_matmul_split_k_reduce;202 203 vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_f16[GGML_TYPE_COUNT];204 vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat[GGML_TYPE_COUNT];205 206 vk_matmul_pipeline pipeline_matmul_id_f32 {};207 vk_matmul_pipeline2 pipeline_matmul_id_f16;208 vk_matmul_pipeline2 pipeline_matmul_id_f16_f32;209 210 vk_matmul_pipeline2 pipeline_dequant_mul_mat_mat_id[GGML_TYPE_COUNT];211 212 vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];213 vk_pipeline pipeline_dequant_mul_mat_vec_f32_f32[GGML_TYPE_COUNT][mul_mat_vec_max_cols];214 vk_pipeline pipeline_dequant_mul_mat_vec_f16_f32[GGML_TYPE_COUNT][mul_mat_vec_max_cols];215 vk_pipeline pipeline_dequant_mul_mat_vec_id_f32[GGML_TYPE_COUNT];216 217 vk_pipeline pipeline_mul_mat_vec_p021_f16_f32;218 vk_pipeline pipeline_mul_mat_vec_nc_f16_f32;219 vk_pipeline pipeline_get_rows[GGML_TYPE_COUNT];220 vk_pipeline pipeline_get_rows_f32[GGML_TYPE_COUNT];221 vk_pipeline pipeline_acc_f32;222 vk_pipeline pipeline_add_f32, pipeline_add_f32_norepeat;223 vk_pipeline pipeline_add_f16_f32_f16, pipeline_add_f16_f32_f16_norepeat;224 vk_pipeline pipeline_mul_f32, pipeline_mul_f32_norepeat;225 vk_pipeline pipeline_div_f32, pipeline_div_f32_norepeat;226 vk_pipeline pipeline_concat_f32, pipeline_concat_f16, pipeline_concat_i32;227 vk_pipeline pipeline_upscale_f32;228 vk_pipeline pipeline_scale_f32;229 vk_pipeline pipeline_sqr_f32;230 vk_pipeline pipeline_sin_f32;231 vk_pipeline pipeline_cos_f32;232 vk_pipeline pipeline_clamp_f32;233 vk_pipeline pipeline_pad_f32;234 vk_pipeline pipeline_repeat_f32;235 vk_pipeline pipeline_cpy_f32_f32, pipeline_cpy_f32_f16, pipeline_cpy_f16_f16;236 vk_pipeline pipeline_contig_cpy_f32_f32, pipeline_contig_cpy_f32_f16, pipeline_contig_cpy_f16_f16;237 vk_pipeline pipeline_cpy_f32_quant[GGML_TYPE_COUNT];238 vk_pipeline pipeline_cpy_quant_f32[GGML_TYPE_COUNT];239 vk_pipeline pipeline_norm_f32;240 vk_pipeline pipeline_group_norm_f32;241 vk_pipeline pipeline_rms_norm_f32;242 vk_pipeline pipeline_gelu_f32;243 vk_pipeline pipeline_gelu_quick_f32;244 vk_pipeline pipeline_silu_f32;245 vk_pipeline pipeline_relu_f32;246 vk_pipeline pipeline_leaky_relu_f32;247 vk_pipeline pipeline_tanh_f32;248 vk_pipeline pipeline_diag_mask_inf_f32;249 vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;250 vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;251 vk_pipeline pipeline_rope_norm_f32, pipeline_rope_norm_f16;252 vk_pipeline pipeline_rope_neox_f32, pipeline_rope_neox_f16;253 vk_pipeline pipeline_argsort_f32;254 vk_pipeline pipeline_sum_rows_f32;255 vk_pipeline pipeline_im2col_f32, pipeline_im2col_f32_f16;256 vk_pipeline pipeline_timestep_embedding_f32;257 vk_pipeline pipeline_pool2d_f32;258 vk_pipeline pipeline_rwkv_wkv6_f32;259 260 // [2][2][2] is for {f16acc,f32acc}x{large,small_rows}x{unaligned, aligned}261 vk_pipeline pipeline_flash_attn_f32_f16_D64[GGML_TYPE_COUNT][2][2][2];262 vk_pipeline pipeline_flash_attn_f32_f16_D80[GGML_TYPE_COUNT][2][2][2];263 vk_pipeline pipeline_flash_attn_f32_f16_D96[GGML_TYPE_COUNT][2][2][2];264 vk_pipeline pipeline_flash_attn_f32_f16_D112[GGML_TYPE_COUNT][2][2][2];265 vk_pipeline pipeline_flash_attn_f32_f16_D128[GGML_TYPE_COUNT][2][2][2];266 vk_pipeline pipeline_flash_attn_f32_f16_D256[GGML_TYPE_COUNT][2][2][2];267 268 std::unordered_map<std::string, vk_pipeline_ref> pipelines;269 std::unordered_map<std::string, uint64_t> pipeline_descriptor_set_requirements;270 271 std::vector<std::tuple<void*, size_t, vk_buffer>> pinned_memory;272 273 vk::Fence fence;274 vk_buffer sync_staging;275 276 ggml_backend_buffer_type buffer_type;277 278#ifdef GGML_VULKAN_MEMORY_DEBUG279 std::unique_ptr<vk_memory_logger> memory_logger;280#endif281#ifdef GGML_VULKAN_PERF282 std::unique_ptr<vk_perf_logger> perf_logger;283#endif284 285 ~vk_device_struct() {286 VK_LOG_DEBUG("destroy device " << name);287 288 device.destroyFence(fence);289 290 ggml_vk_destroy_buffer(sync_staging);291 292 device.destroyCommandPool(compute_queue.pool);293 if (!single_queue) {294 device.destroyCommandPool(transfer_queue.pool);295 }296 297 for (auto& pipeline : pipelines) {298 if (pipeline.second.expired()) {299 continue;300 }301 302 vk_pipeline pl = pipeline.second.lock();303 ggml_vk_destroy_pipeline(device, pl);304 }305 pipelines.clear();306 307 device.destroy();308 }309};310 311struct vk_buffer_struct {312 vk::Buffer buffer = VK_NULL_HANDLE;313 vk::DeviceMemory device_memory = VK_NULL_HANDLE;314 vk::MemoryPropertyFlags memory_property_flags;315 void * ptr;316 size_t size = 0;317 318 vk_device device;319 320 ~vk_buffer_struct() {321 if (size == 0) {322 return;323 }324 VK_LOG_DEBUG("~vk_buffer_struct(" << buffer << ", " << size << ")");325 326 device->device.freeMemory(device_memory);327 device->device.destroyBuffer(buffer);328 }329};330 331struct vk_subbuffer {332 vk_buffer buffer;333 uint64_t offset;334 uint64_t size;335 336 operator vk::DescriptorBufferInfo() const {337 return { buffer->buffer, offset, size };338 }339};340 341struct vk_semaphore {342 vk::Semaphore s;343 uint64_t value;344};345 346struct vk_submission {347 vk::CommandBuffer buffer;348 std::vector<vk_semaphore> wait_semaphores;349 std::vector<vk_semaphore> signal_semaphores;350};351 352typedef std::vector<vk_submission> vk_sequence;353 354struct vk_mat_mat_push_constants {355 uint32_t M; uint32_t N; uint32_t K;356 uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;357 uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;358 uint32_t k_split;359 uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;360};361struct vk_mat_vec_push_constants {362 uint32_t ncols; uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;363 uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;364 uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;365};366 367struct vk_mat_mat_id_push_constants {368 uint32_t M; uint32_t N; uint32_t K;369 uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;370 uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;371 uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;372};373struct vk_mat_vec_id_push_constants {374 uint32_t ncols; uint32_t stride_a; uint32_t stride_b; uint32_t stride_d;375 uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;376 uint32_t nei0; uint32_t ne11;377};378 379struct vk_flash_attn_push_constants {380 uint32_t N;381 uint32_t KV;382 383 uint32_t ne1;384 uint32_t ne2;385 uint32_t ne3;386 387 uint32_t neq2;388 uint32_t neq3;389 uint32_t nek2;390 uint32_t nek3;391 uint32_t nev2;392 uint32_t nev3;393 uint32_t nem1;394 395 uint32_t nb01;396 uint32_t nb02;397 uint32_t nb03;398 uint32_t nb11;399 uint32_t nb12;400 uint32_t nb13;401 uint32_t nb21;402 uint32_t nb22;403 uint32_t nb23;404 uint32_t nb31;405 406 float scale;407 float max_bias;408 float logit_softcap;409 410 uint32_t mask;411 uint32_t n_head_log2;412 float m0;413 float m1;414};415 416struct vk_op_push_constants {417 uint32_t KX;418 uint32_t KY;419 float param1;420 float param2;421};422 423struct vk_op_unary_push_constants {424 uint32_t ne;425 uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;426 uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;427 uint32_t misalign_offsets;428 float param1; float param2;429 uint32_t ne0_012mp; uint32_t ne0_012L;430 uint32_t ne0_01mp; uint32_t ne0_01L;431 uint32_t ne0_0mp; uint32_t ne0_0L;432 uint32_t ne1_012mp; uint32_t ne1_012L;433 uint32_t ne1_01mp; uint32_t ne1_01L;434 uint32_t ne1_0mp; uint32_t ne1_0L;435};436static_assert(sizeof(vk_op_unary_push_constants) <= 128, "sizeof(vk_op_unary_push_constants) must be <= 128");437 438// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.439// Precompute mp (m' in the paper) and L such that division440// can be computed using a multiply (high 32b of 64b result)441// and a shift:442//443// n/d = (mulhi(n, mp) + n) >> L;444static void init_fastdiv_values(uint32_t d, uint32_t &mp, uint32_t &L)445{446 // compute L = ceil(log2(d));447 L = 0;448 while (L < 32 && (uint32_t{1} << L) < d) {449 L++;450 }451 452 mp = (uint32_t)((uint64_t{1} << 32) * ((uint64_t{1} << L) - d) / d + 1);453}454 455template <typename T> void init_pushconst_fastdiv(T &p) {456 GGML_UNUSED(p);457 static_assert(!std::is_const<T>::value, "unexpected type");458}459 460template <> void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {461 // Compute magic values to divide by these six numbers.462 init_fastdiv_values(p.ne02*p.ne01*p.ne00, p.ne0_012mp, p.ne0_012L);463 init_fastdiv_values(p.ne01*p.ne00, p.ne0_01mp, p.ne0_01L);464 init_fastdiv_values(p.ne00, p.ne0_0mp, p.ne0_0L);465 init_fastdiv_values(p.ne12*p.ne11*p.ne10, p.ne1_012mp, p.ne1_012L);466 init_fastdiv_values(p.ne11*p.ne10, p.ne1_01mp, p.ne1_01L);467 init_fastdiv_values(p.ne10, p.ne1_0mp, p.ne1_0L);468}469 470struct vk_op_binary_push_constants {471 uint32_t ne;472 uint32_t ne00; uint32_t ne01; uint32_t ne02; uint32_t ne03; uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;473 uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13; uint32_t nb10; uint32_t nb11; uint32_t nb12; uint32_t nb13;474 uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; uint32_t nb20; uint32_t nb21; uint32_t nb22; uint32_t nb23;475 uint32_t misalign_offsets;476 float param1; float param2; int32_t param3;477};478 479struct vk_op_diag_mask_push_constants {480 uint32_t ncols;481 uint32_t rows_per_channel;482 int32_t n_past;483};484 485struct vk_op_rope_push_constants {486 uint32_t ncols;487 uint32_t n_dims;488 float freq_scale;489 uint32_t p_delta_rows;490 float freq_base;491 float ext_factor;492 float attn_factor;493 float corr_dims[2];494 float theta_scale;495 uint32_t has_ff;496};497 498struct vk_op_soft_max_push_constants {499 uint32_t KX;500 uint32_t KY;501 float scale;502 float max_bias;503 float m0;504 float m1;505 uint32_t n_head_log2;506 uint32_t nrows_x;507};508 509struct vk_op_argsort_push_constants {510 uint32_t ncols;511 uint32_t ncols_pad;512 int32_t order;513};514 515struct vk_op_im2col_push_constants {516 uint32_t batch_offset; uint32_t offset_delta;517 uint32_t IC;518 uint32_t IW; uint32_t IH;519 uint32_t OW; uint32_t OH;520 uint32_t KW; uint32_t KH;521 uint32_t pelements;522 uint32_t CHW;523 int32_t s0; int32_t s1;524 int32_t p0; int32_t p1;525 int32_t d0; int32_t d1;526};527 528struct vk_op_timestep_embedding_push_constants {529 uint32_t nb1;530 uint32_t dim;531 uint32_t max_period;532};533 534struct vk_op_pool2d_push_constants {535 uint32_t IW; uint32_t IH;536 uint32_t OW; uint32_t OH;537 uint32_t OC;538 uint32_t pelements;539 uint32_t op;540 int32_t k0; int32_t k1;541 int32_t s0; int32_t s1;542 int32_t p0; int32_t p1;543};544 545struct vk_op_rwkv_wkv6_push_constants {546 uint32_t B;547 uint32_t T;548 uint32_t C;549 uint32_t H;550};551 552// Allow pre-recording command buffers553struct vk_staging_memcpy {554 vk_staging_memcpy(void * _dst, const void * _src, size_t _n) : dst(_dst), src(_src), n(_n) {}555 556 void * dst;557 const void * src;558 size_t n;559};560 561struct vk_op_upscale_push_constants {562 uint32_t ne; uint32_t a_offset; uint32_t d_offset;563 uint32_t nb00; uint32_t nb01; uint32_t nb02; uint32_t nb03;564 uint32_t ne10; uint32_t ne11; uint32_t ne12; uint32_t ne13;565 float sf0; float sf1; float sf2; float sf3;566};567 568struct vk_context_struct {569 vk_submission * s;570 std::vector<vk_sequence> seqs;571 572 int exit_tensor_idx;573 574 std::vector<vk_staging_memcpy> in_memcpys;575 std::vector<vk_staging_memcpy> out_memcpys;576 577 vk_queue * q;578};579typedef std::shared_ptr<vk_context_struct> vk_context;580typedef std::weak_ptr<vk_context_struct> vk_context_ref;581 582struct ggml_vk_garbage_collector {583 std::vector<vk_semaphore> tl_semaphores;584 std::vector<vk_semaphore> semaphores;585 std::vector<vk::Event> events;586 std::vector<vk_buffer> temp_buffers;587 std::vector<vk_context> contexts;588};589 590#if defined(GGML_VULKAN_MEMORY_DEBUG) || defined(GGML_VULKAN_DEBUG)591#define VK_LOG_MEMORY(msg) std::cerr << "ggml_vulkan memory: " << msg << std::endl592 593static std::string format_size(size_t size) {594 const size_t kib = 1024;595 const size_t mib = kib * 1024;596 const size_t gib = mib * 1024;597 598 std::ostringstream oss;599 oss << std::fixed << std::setprecision(2);600 601 if (size >= gib) {602 oss << static_cast<double>(size) / gib << " GiB";603 } else if (size >= mib) {604 oss << static_cast<double>(size) / mib << " MiB";605 } else if (size >= kib) {606 oss << static_cast<double>(size) / kib << " KiB";607 } else {608 oss << size << " B";609 }610 611 return oss.str();612}613 614static std::mutex log_mutex;615 616class vk_memory_logger {617public:618 vk_memory_logger(): total_device(0), total_host(0) {}619 void log_allocation(vk_buffer_ref buf_ref, size_t size);620 void log_deallocation(vk_buffer_ref buf_ref);621 622private:623 std::map<vk::Buffer, size_t> allocations; // Track allocations624 size_t total_device;625 size_t total_host;626};627#else628#define VK_LOG_MEMORY(msg) ((void) 0)629#endif // GGML_VULKAN_MEMORY_DEBUG630 631#if defined(GGML_VULKAN_PERF)632 633class vk_perf_logger {634public:635 void print_timings() {636 std::cerr << "----------------\nVulkan Timings:" << std::endl;637 for (const auto& t : timings) {638 uint64_t total = 0;639 for (const auto& time : t.second) {640 total += time;641 }642 std::cerr << t.first << ": " << t.second.size() << " x " << (total / t.second.size() / 1000.0) << " ms" << std::endl;643 }644 645 timings.clear();646 }647 648 void log_timing(const ggml_tensor * node, uint64_t time) {649 if (node->op == GGML_OP_UNARY) {650 timings[ggml_unary_op_name(ggml_get_unary_op(node))].push_back(time);651 return;652 }653 if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) {654 const uint64_t m = node->src[0]->ne[1];655 const uint64_t n = node->src[1]->ne[1];656 const uint64_t k = node->src[1]->ne[0];657 std::string name = ggml_op_name(node->op);658 if (n == 1) {659 name += "_VEC m=" + std::to_string(m) + " k=" + std::to_string(k);660 } else {661 name += " m=" + std::to_string(m) + " n=" + std::to_string(n) + " k=" + std::to_string(k);662 }663 timings[name].push_back(time);664 return;665 }666 timings[ggml_op_name(node->op)].push_back(time);667 }668private:669 std::map<std::string, std::vector<uint64_t>> timings;670};671#endif // GGML_VULKAN_PERF672 673struct ggml_backend_vk_context {674 std::string name;675 676 vk_device device;677 678 size_t semaphore_idx, event_idx;679 ggml_vk_garbage_collector gc;680 size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k;681 vk_buffer prealloc_x, prealloc_y, prealloc_split_k;682 vk::Fence fence;683 684 vk_buffer buffer_pool[MAX_VK_BUFFERS];685 686 vk_context_ref compute_ctx;687 vk_context_ref transfer_ctx;688 689 std::vector<vk_context_ref> tensor_ctxs;690};691 692static void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT693 694static uint64_t vk_tensor_offset(const ggml_tensor * tensor) {695 if (tensor->view_src) {696 return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base;697 }698 return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base;699}700 701struct ggml_backend_vk_buffer_context {702 vk_device_ref device;703 vk_buffer dev_buffer;704 std::string name;705 706 ggml_backend_vk_buffer_context(vk_device_ref device, vk_buffer&& dev_buffer, std::string& name) :707 device(device),708 dev_buffer(dev_buffer),709 name(name) {710 }711 712 ~ggml_backend_vk_buffer_context() {713 ggml_vk_destroy_buffer(dev_buffer);714 }715};716 717#ifdef GGML_VULKAN_MEMORY_DEBUG718void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {719 std::lock_guard<std::mutex> guard(log_mutex);720 vk_buffer buf = buf_ref.lock();721 const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);722 const std::string type = device ? "device" : "host";723 allocations[buf->buffer] = size;724 total_device += device ? size : 0;725 total_host += device ? 0 : size;726 VK_LOG_MEMORY(buf->device->name << ": +" << format_size(size) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));727}728 729void vk_memory_logger::log_deallocation(vk_buffer_ref buf_ref) {730 if (buf_ref.expired() || buf_ref.lock()->size == 0) {731 return;732 }733 734 std::lock_guard<std::mutex> guard(log_mutex);735 vk_buffer buf = buf_ref.lock();736 const bool device = bool(buf->memory_property_flags & vk::MemoryPropertyFlagBits::eDeviceLocal);737 std::string type = device ? "device" : "host";738 auto it = allocations.find(buf->buffer);739 total_device -= device ? it->second : 0;740 total_host -= device ? 0 : it->second;741 if (it != allocations.end()) {742 VK_LOG_MEMORY(buf->device->name << ": -" << format_size(it->second) << " " << type << " at " << buf->buffer << ". Total device: " << format_size(total_device) << ", total host: " << format_size(total_host));743 allocations.erase(it);744 } else {745 VK_LOG_MEMORY("ERROR " << buf->device->name << ": Attempted to deallocate unknown " << type << " memory at " << buf->buffer);746 }747}748#endif // GGML_VULKAN_MEMORY_DEBUG749 750struct vk_instance_t {751 vk::Instance instance;752 753 std::vector<size_t> device_indices;754 vk_device devices[GGML_VK_MAX_DEVICES];755};756 757static bool vk_instance_initialized = false;758static vk_instance_t vk_instance;759 760#ifdef GGML_VULKAN_CHECK_RESULTS761static size_t vk_skip_checks;762static size_t vk_output_tensor;763 764static void ggml_vk_print_tensor(const ggml_tensor * tensor, const char * name);765static void ggml_vk_check_results_0(ggml_tensor * tensor);766static void ggml_vk_check_results_1(ggml_tensor * tensor);767#endif768 769typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);770 771static void ggml_backend_vk_free(ggml_backend_t backend);772 773// variables to track number of compiles in progress774static uint32_t compile_count = 0;775static std::mutex compile_count_mutex;776static std::condition_variable compile_count_cond;777 778static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipeline, size_t spv_size, const void* spv_data, const std::string entrypoint,779 uint32_t parameter_count, std::array<uint32_t, 3> wg_denoms, std::vector<uint32_t> specialization_constants,780 bool disable_robustness, bool require_full_subgroups, uint32_t required_subgroup_size) {781 VK_LOG_DEBUG("ggml_vk_create_pipeline(" << device->name << ", " << pipeline->name << ", " << entrypoint << ", " << parameter_count <<782 ", (" << wg_denoms[0] << "," << wg_denoms[1] << "," << wg_denoms[2] << "), specialization_constants, " <<783 disable_robustness << ", " << require_full_subgroups << ", " << required_subgroup_size << ")");784 GGML_ASSERT(parameter_count > 0);785 GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT786 787 vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast<const uint32_t *>(spv_data));788 pipeline->shader_module = device->device.createShaderModule(shader_module_create_info);789 790 std::vector<vk::DescriptorSetLayoutBinding> dsl_binding;791 std::vector<vk::DescriptorBindingFlags> dsl_binding_flags;792 for (uint32_t i = 0; i < parameter_count; i++) {793 dsl_binding.push_back({i, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute});794 dsl_binding_flags.push_back({});795 }796 797 vk::DescriptorSetLayoutBindingFlagsCreateInfo dslbfci = { dsl_binding_flags };798 799 vk::PushConstantRange pcr(800 vk::ShaderStageFlagBits::eCompute,801 0,802 pipeline->push_constant_size803 );804 805 vk::DescriptorSetLayoutCreateInfo descriptor_set_layout_create_info(806 {},807 dsl_binding);808 descriptor_set_layout_create_info.setPNext(&dslbfci);809 pipeline->dsl = device->device.createDescriptorSetLayout(descriptor_set_layout_create_info);810 811 vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline->parameter_count * VK_DEVICE_DESCRIPTOR_POOL_SIZE);812 vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, VK_DEVICE_DESCRIPTOR_POOL_SIZE, descriptor_pool_size);813 pipeline->descriptor_pools.push_back(device->device.createDescriptorPool(descriptor_pool_create_info));814 815 pipeline->descriptor_set_idx = 0;816 817 vk::PipelineLayoutCreateInfo pipeline_layout_create_info(vk::PipelineLayoutCreateFlags(), pipeline->dsl, pcr);818 pipeline->layout = device->device.createPipelineLayout(pipeline_layout_create_info);819 820 std::vector<vk::SpecializationMapEntry> specialization_entries(specialization_constants.size());821 822 for (size_t i = 0; i < specialization_constants.size(); i++) {823 specialization_entries[i].constantID = i;824 specialization_entries[i].offset = i * sizeof(uint32_t);825 specialization_entries[i].size = sizeof(uint32_t);826 }827 828 vk::SpecializationInfo specialization_info(829 specialization_entries.size(),830 specialization_entries.data(),831 specialization_constants.size() * sizeof(uint32_t),832 specialization_constants.data()833 );834 835 vk::PipelineShaderStageCreateFlags pipeline_shader_stage_create_flags{};836 837 if (device->subgroup_require_full_support && require_full_subgroups) {838 pipeline_shader_stage_create_flags |= vk::PipelineShaderStageCreateFlagBits::eRequireFullSubgroupsEXT;839 }840 841 vk::PipelineShaderStageCreateInfo pipeline_shader_create_info(842 pipeline_shader_stage_create_flags,843 vk::ShaderStageFlagBits::eCompute,844 pipeline->shader_module,845 entrypoint.c_str(),846 &specialization_info);847 848 vk::PipelineShaderStageRequiredSubgroupSizeCreateInfoEXT pipeline_shader_stage_required_subgroup_size_create_info;849 pipeline_shader_stage_required_subgroup_size_create_info.requiredSubgroupSize = required_subgroup_size;850 if (device->subgroup_size_control && required_subgroup_size > 0) {851 GGML_ASSERT(device->subgroup_min_size <= required_subgroup_size && required_subgroup_size <= device->subgroup_max_size);852 pipeline_shader_create_info.setPNext(&pipeline_shader_stage_required_subgroup_size_create_info);853 }854 855 vk::ComputePipelineCreateInfo compute_pipeline_create_info(856 vk::PipelineCreateFlags{},857 pipeline_shader_create_info,858 pipeline->layout);859 860 vk::PipelineRobustnessCreateInfoEXT rci;861 862 if (device->pipeline_robustness && disable_robustness) {863 rci.storageBuffers = vk::PipelineRobustnessBufferBehaviorEXT::eDisabled;864 rci.uniformBuffers = vk::PipelineRobustnessBufferBehaviorEXT::eDisabled;865 compute_pipeline_create_info.setPNext(&rci);866 }867 868 try {869 pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value;870 } catch (const vk::SystemError& e) {871 std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl;872 std::cerr << "ggml_vulkan: " << e.what() << std::endl;873 throw e;874 }875 pipeline->compiled = true;876 877 {878 std::lock_guard<std::mutex> guard(device->mutex);879 device->pipelines.insert({ pipeline->name, pipeline });880 }881 882 {883 std::lock_guard<std::mutex> guard(compile_count_mutex);884 assert(compile_count > 0);885 compile_count--;886 }887 compile_count_cond.notify_all();888}889 890static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {891 VK_LOG_DEBUG("ggml_pipeline_destroy_pipeline(" << pipeline->name << ")");892 for (auto& pool : pipeline->descriptor_pools) {893 device.destroyDescriptorPool(pool);894 }895 pipeline->descriptor_pools.clear();896 pipeline->descriptor_sets.clear();897 pipeline->descriptor_set_idx = 0;898 899 device.destroyDescriptorSetLayout(pipeline->dsl);900 901 device.destroyPipelineLayout(pipeline->layout);902 903 device.destroyShaderModule(pipeline->shader_module);904 905 device.destroyPipeline(pipeline->pipeline);906}907 908static void ggml_pipeline_request_descriptor_sets(vk_device& device, vk_pipeline& pipeline, uint32_t n) {909 VK_LOG_DEBUG("ggml_pipeline_request_descriptor_sets(" << pipeline->name << ", " << n << ")");910 device->pipeline_descriptor_set_requirements[pipeline->name] += n;911 if (!pipeline->compiled) {912 pipeline->needed = true;913 device->need_compiles = true;914 }915}916 917static void ggml_pipeline_allocate_descriptor_sets(vk_device& device) {918 std::lock_guard<std::mutex> guard(device->mutex);919 920 for (auto& pair : device->pipeline_descriptor_set_requirements) {921 vk_pipeline pipeline = device->pipelines.at(pair.first).lock();922 const uint64_t n = pair.second;923 924 VK_LOG_DEBUG("ggml_pipeline_allocate_descriptor_sets(" << pipeline->name << ", " << n << ")");925 926 if (pipeline->descriptor_sets.size() >= pipeline->descriptor_set_idx + n) {927 // Enough descriptors are available928 continue;929 }930 931 uint32_t to_alloc = pipeline->descriptor_set_idx + n - pipeline->descriptor_sets.size();932 uint32_t pool_remaining = VK_DEVICE_DESCRIPTOR_POOL_SIZE - pipeline->descriptor_sets.size() % VK_DEVICE_DESCRIPTOR_POOL_SIZE;933 uint32_t pool_idx = pipeline->descriptor_sets.size() / VK_DEVICE_DESCRIPTOR_POOL_SIZE;934 935 while (to_alloc > 0) {936 const uint32_t alloc_count = std::min(pool_remaining, to_alloc);937 to_alloc -= alloc_count;938 pool_remaining = VK_DEVICE_DESCRIPTOR_POOL_SIZE;939 940 if (pool_idx >= pipeline->descriptor_pools.size()) {941 vk::DescriptorPoolSize descriptor_pool_size(vk::DescriptorType::eStorageBuffer, pipeline->parameter_count * VK_DEVICE_DESCRIPTOR_POOL_SIZE);942 vk::DescriptorPoolCreateInfo descriptor_pool_create_info({}, VK_DEVICE_DESCRIPTOR_POOL_SIZE, descriptor_pool_size);943 pipeline->descriptor_pools.push_back(device->device.createDescriptorPool(descriptor_pool_create_info));944 }945 946 std::vector<vk::DescriptorSetLayout> layouts(alloc_count);947 for (uint32_t i = 0; i < alloc_count; i++) {948 layouts[i] = pipeline->dsl;949 }950 vk::DescriptorSetAllocateInfo descriptor_set_alloc_info(pipeline->descriptor_pools[pool_idx], alloc_count, layouts.data());951 std::vector<vk::DescriptorSet> sets = device->device.allocateDescriptorSets(descriptor_set_alloc_info);952 pipeline->descriptor_sets.insert(pipeline->descriptor_sets.end(), sets.begin(), sets.end());953 954 pool_idx++;955 }956 }957}958 959static void ggml_pipeline_cleanup(vk_pipeline& pipeline) {960 VK_LOG_DEBUG("ggml_pipeline_cleanup(" << pipeline->name << ")");961 pipeline->descriptor_set_idx = 0;962}963 964static vk::CommandBuffer ggml_vk_create_cmd_buffer(vk_device& device, vk_queue& q) {965 VK_LOG_DEBUG("ggml_vk_create_cmd_buffer()");966 std::lock_guard<std::mutex> guard(device->mutex);967 968 if (q.cmd_buffers.size() > q.cmd_buffer_idx) {969 // Reuse command buffer970 return q.cmd_buffers[q.cmd_buffer_idx++];971 }972 973 vk::CommandBufferAllocateInfo command_buffer_alloc_info(974 q.pool,975 vk::CommandBufferLevel::ePrimary,976 1);977 const std::vector<vk::CommandBuffer> cmd_buffers = device->device.allocateCommandBuffers(command_buffer_alloc_info);978 auto buf = cmd_buffers.front();979 980 q.cmd_buffers.push_back(buf);981 q.cmd_buffer_idx++;982 983 return buf;984}985 986static vk_submission ggml_vk_create_submission(vk_device& device, vk_queue& q, std::vector<vk_semaphore> wait_semaphores, std::vector<vk_semaphore> signal_semaphores) {987 VK_LOG_DEBUG("ggml_vk_create_submission()");988 vk_submission s;989 s.buffer = ggml_vk_create_cmd_buffer(device, q);990 s.wait_semaphores = std::move(wait_semaphores);991 s.signal_semaphores = std::move(signal_semaphores);992 return s;993}994 995static void ggml_vk_submit(vk_context& ctx, vk::Fence fence) {996 if (ctx->seqs.empty()) {997 if (fence) {998 ctx->q->queue.submit({}, fence);999 }1000 return;1001 }1002 VK_LOG_DEBUG("ggml_vk_submit(" << ctx << ", " << fence << ")");1003 1004 std::vector<std::vector<uint64_t>> tl_wait_vals;1005 std::vector<std::vector<uint64_t>> tl_signal_vals;1006 std::vector<std::vector<vk::Semaphore>> tl_wait_semaphores;1007 std::vector<std::vector<vk::Semaphore>> tl_signal_semaphores;1008 std::vector<vk::TimelineSemaphoreSubmitInfo> tl_submit_infos;1009 std::vector<vk::SubmitInfo> submit_infos;1010 int idx = -1;1011 std::vector<std::vector<vk::PipelineStageFlags>> stage_flags;1012 1013 size_t reserve = 0;1014 1015 for (const auto& sequence : ctx->seqs) {1016 reserve += sequence.size();1017 }1018 1019 // Pre-reserve vectors to prevent reallocation, which invalidates pointers1020 tl_wait_semaphores.reserve(reserve);1021 tl_wait_vals.reserve(reserve);1022 tl_signal_semaphores.reserve(reserve);1023 tl_signal_vals.reserve(reserve);1024 tl_submit_infos.reserve(reserve);1025 submit_infos.reserve(reserve);1026 stage_flags.reserve(reserve);1027 1028 for (const auto& sequence : ctx->seqs) {1029 for (const auto& submission : sequence) {1030 stage_flags.push_back({});1031 idx++;1032 tl_wait_vals.push_back({});1033 tl_wait_semaphores.push_back({});1034 tl_signal_vals.push_back({});1035 tl_signal_semaphores.push_back({});1036 for (size_t i = 0; i < submission.wait_semaphores.size(); i++) {1037 stage_flags[idx].push_back(ctx->q->stage_flags);1038 tl_wait_vals[idx].push_back(submission.wait_semaphores[i].value);1039 tl_wait_semaphores[idx].push_back(submission.wait_semaphores[i].s);1040 }1041 for (size_t i = 0; i < submission.signal_semaphores.size(); i++) {1042 tl_signal_vals[idx].push_back(submission.signal_semaphores[i].value);1043 tl_signal_semaphores[idx].push_back(submission.signal_semaphores[i].s);1044 }1045 tl_submit_infos.push_back({1046 (uint32_t) submission.wait_semaphores.size(),1047 tl_wait_vals[idx].data(),1048 (uint32_t) submission.signal_semaphores.size(),1049 tl_signal_vals[idx].data(),1050 });1051 tl_submit_infos[idx].sType = vk::StructureType::eTimelineSemaphoreSubmitInfo;1052 tl_submit_infos[idx].pNext = nullptr;1053 vk::SubmitInfo si{1054 (uint32_t) submission.wait_semaphores.size(),1055 tl_wait_semaphores[idx].data(),1056 stage_flags[idx].data(),1057 1,1058 &submission.buffer,1059 (uint32_t) submission.signal_semaphores.size(),1060 tl_signal_semaphores[idx].data(),1061 };1062 si.setPNext(&tl_submit_infos[idx]);1063 submit_infos.push_back(si);1064 }1065 }1066 1067 ctx->q->queue.submit(submit_infos, fence);1068 1069 ctx->seqs.clear();1070}1071 1072static uint32_t ggml_vk_find_queue_family_index(std::vector<vk::QueueFamilyProperties>& queue_family_props, const vk::QueueFlags& required, const vk::QueueFlags& avoid, int32_t compute_index, uint32_t min_num_queues) {1073 VK_LOG_DEBUG("ggml_vk_find_queue_family_index()");1074 const uint32_t qfsize = queue_family_props.size();1075 1076 // Try with avoid preferences first1077 for (uint32_t i = 0; i < qfsize; i++) {1078 if (queue_family_props[i].queueCount >= min_num_queues && (compute_index < 0 || i != (uint32_t) compute_index) && queue_family_props[i].queueFlags & required && !(queue_family_props[i].queueFlags & avoid)) {1079 return i;1080 }1081 }1082 1083 // Fall back to only required1084 for (size_t i = 0; i < qfsize; i++) {1085 if (queue_family_props[i].queueCount >= min_num_queues && (compute_index < 0 || i != (uint32_t) compute_index) && queue_family_props[i].queueFlags & required) {1086 return i;1087 }1088 }1089 1090 // Fall back to reusing compute queue1091 for (size_t i = 0; i < qfsize; i++) {1092 if (queue_family_props[i].queueCount >= min_num_queues && queue_family_props[i].queueFlags & required) {1093 return i;1094 }1095 }1096 1097 // Fall back to ignoring min_num_queries1098 for (size_t i = 0; i < qfsize; i++) {1099 if (queue_family_props[i].queueFlags & required) {1100 return i;1101 }1102 }1103 1104 // All commands that are allowed on a queue that supports transfer operations are also allowed on a queue that supports either graphics or compute operations.1105 // Thus, if the capabilities of a queue family include VK_QUEUE_GRAPHICS_BIT or VK_QUEUE_COMPUTE_BIT, then reporting the VK_QUEUE_TRANSFER_BIT capability separately for that queue family is optional.1106 if (compute_index >= 0) {1107 return compute_index;1108 }1109 1110 std::cerr << "ggml_vulkan: No suitable queue family index found." << std::endl;1111 1112 for(auto &q_family : queue_family_props) {1113 std::cerr << "Queue number: " + std::to_string(q_family.queueCount) << " flags: " + to_string(q_family.queueFlags) << std::endl;1114 }1115 abort();1116}1117 1118static void ggml_vk_create_queue(vk_device& device, vk_queue& q, uint32_t queue_family_index, uint32_t queue_index, vk::PipelineStageFlags&& stage_flags, bool transfer_only) {1119 VK_LOG_DEBUG("ggml_vk_create_queue()");1120 std::lock_guard<std::mutex> guard(device->mutex);1121 1122 q.queue_family_index = queue_family_index;1123 q.transfer_only = transfer_only;1124 1125 vk::CommandPoolCreateInfo command_pool_create_info_compute(vk::CommandPoolCreateFlags(VK_COMMAND_POOL_CREATE_TRANSIENT_BIT), queue_family_index);1126 q.pool = device->device.createCommandPool(command_pool_create_info_compute);1127 1128 q.cmd_buffer_idx = 0;1129 1130 q.queue = device->device.getQueue(queue_family_index, queue_index);1131 1132 q.stage_flags = stage_flags;1133}1134 1135static vk_context ggml_vk_create_context(ggml_backend_vk_context * ctx, vk_queue& q) {1136 vk_context result = std::make_shared<vk_context_struct>();1137 VK_LOG_DEBUG("ggml_vk_create_context(" << result << ")");1138 ctx->gc.contexts.emplace_back(result);1139 result->q = &q;1140 return result;1141}1142 1143static vk_context ggml_vk_create_temporary_context(vk_queue& q) {1144 vk_context result = std::make_shared<vk_context_struct>();1145 VK_LOG_DEBUG("ggml_vk_create_temporary_context(" << result << ")");1146 result->q = &q;1147 return result;1148}1149 1150static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) {1151 VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()");1152 vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 };1153 vk::SemaphoreCreateInfo ci{};1154 ci.setPNext(&tci);1155 vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci);1156 ctx->gc.semaphores.push_back({ semaphore, 0 });1157 return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1];1158}1159 1160static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) {1161 VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()");1162 if (ctx->semaphore_idx >= ctx->gc.tl_semaphores.size()) {1163 vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eTimeline, 0 };1164 vk::SemaphoreCreateInfo ci{};1165 ci.setPNext(&tci);1166 vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci);1167 ctx->gc.tl_semaphores.push_back({ semaphore, 0 });1168 }1169 return &ctx->gc.tl_semaphores[ctx->semaphore_idx++];1170}1171 1172static vk::Event ggml_vk_create_event(ggml_backend_vk_context * ctx) {1173 if (ctx->event_idx >= ctx->gc.events.size()) {1174 ctx->gc.events.push_back(ctx->device->device.createEvent({}));1175 }1176 return ctx->gc.events[ctx->event_idx++];1177}1178 1179static void ggml_vk_queue_cleanup(vk_device& device, vk_queue& q) {1180 VK_LOG_DEBUG("ggml_vk_queue_cleanup()");1181 std::lock_guard<std::mutex> guard(device->mutex);1182 1183 // Requires command buffers to be done1184 device->device.resetCommandPool(q.pool);1185 q.cmd_buffer_idx = 0;1186}1187 1188static uint32_t find_properties(const vk::PhysicalDeviceMemoryProperties* mem_props, vk::MemoryRequirements* mem_req, vk::MemoryPropertyFlags flags) {1189 for (uint32_t i = 0; i < mem_props->memoryTypeCount; ++i) {1190 vk::MemoryType memory_type = mem_props->memoryTypes[i];1191 if ((mem_req->memoryTypeBits & ((uint64_t)1 << i)) &&1192 (flags & memory_type.propertyFlags) == flags &&1193 mem_props->memoryHeaps[memory_type.heapIndex].size >= mem_req->size) {1194 return static_cast<int32_t>(i);1195 }1196 }1197 return UINT32_MAX;1198}1199 1200static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, vk::MemoryPropertyFlags req_flags, vk::MemoryPropertyFlags fallback_flags = vk::MemoryPropertyFlags(0)) {