KBaba7/llama.cpp

1#pragma once2 3#include "llama.h"4 5#include <array>6#include <vector>7 8// very similar to llama_batch,9// but has more metadata about sequences10struct llama_ubatch {11    bool equal_seqs;12    // TODO: whole_seqs for embeddings?13 14    uint32_t n_tokens; // total tokens (n_seq_tokens * n_seqs)15    uint32_t n_seq_tokens; // tokens per sequence16    uint32_t n_seqs;17 18    llama_token  *  token;    // [n_tokens]19    float        *  embd;     // [n_embd, n_tokens]20    llama_pos    *  pos;      // [n_tokens]21    int32_t      *  n_seq_id; // [n_seqs]22    llama_seq_id ** seq_id;   // [n_seqs]23    int8_t       *  output;   // [n_tokens]24};25 26struct llama_sbatch_seq {27    int32_t n_seq_id;28 29    llama_seq_id * seq_id;30 31    size_t offset;32    size_t length;33};34 35// sequence-length-aware batch splitting36struct llama_sbatch {37    // tokens left in this batch38    size_t n_tokens;39 40    size_t n_embd;41 42    bool logits_all; // TODO: remove once lctx.logits_all is removed too43 44    // sorted indices into the batch45    std::vector<size_t> ids;46    // batch indices of the output47    std::vector<size_t> out_ids;48    std::vector<llama_sbatch_seq> seq;49 50    const llama_batch * batch = nullptr;51 52    // buffers for the ubatch53    std::vector<llama_token>    ubatch_token;54    std::vector<float>          ubatch_embd;55    std::vector<llama_pos>      ubatch_pos;56    std::vector<int32_t>        ubatch_n_seq_id;57    std::vector<llama_seq_id *> ubatch_seq_id;58    std::vector<int8_t>         ubatch_output;59 60    llama_ubatch reserve_ubatch(size_t n_ubatch, bool has_embd = false);61 62    void add_seq_to_ubatch(llama_ubatch & ubatch, llama_sbatch_seq & seq, size_t length);63 64    // simple split, unknown number of sequences of unequal lengths65    llama_ubatch split_simple(size_t n_ubatch);66 67    // make batches of equal-length sequences68    llama_ubatch split_equal(size_t n_ubatch);69 70    // sequence-wise split71    llama_ubatch split_seq(size_t n_ubatch);72 73    void from_batch(const llama_batch & batch, size_t n_embd, bool simple_split = false, bool logits_all = false);74};75 76// temporary allocate memory for the input batch if needed77struct llama_batch_allocr {78    struct llama_batch batch;79 80    std::array<llama_seq_id, 1> seq_id_0 = { 0 }; // default sequence id81    std::vector<llama_pos>      pos;82    std::vector<int32_t>        n_seq_id;83    std::vector<llama_seq_id *> seq_id;84    std::vector<int8_t>         logits;85 86    // optionally fulfill the batch returned by llama_batch_get_one87    llama_batch_allocr(struct llama_batch in_batch, llama_pos p0);88};89