echodict/llama.cpp

1#include "ngram-mod.h"2 3//4// common_ngram_mod5//6 7common_ngram_mod::common_ngram_mod(uint16_t n, size_t size) : n(n), used(0) {8    entries.resize(size);9 10    reset();11}12 13size_t common_ngram_mod::idx(const entry_t * tokens) const {14    size_t res = 0;15 16    for (size_t i = 0; i < n; ++i) {17        res = res*6364136223846793005ULL + tokens[i];18    }19 20    res = res % entries.size();21 22    return res;23}24 25void common_ngram_mod::add(const entry_t * tokens) {26    const size_t i = idx(tokens);27 28    if (entries[i] == EMPTY) {29        used++;30    }31 32    entries[i] = tokens[n];33}34 35common_ngram_mod::entry_t common_ngram_mod::get(const entry_t * tokens) const {36    const size_t i = idx(tokens);37 38    return entries[i];39}40 41void common_ngram_mod::reset() {42    std::fill(entries.begin(), entries.end(), EMPTY);43    used = 0;44}45 46size_t common_ngram_mod::get_n() const {47    return n;48}49 50size_t common_ngram_mod::get_used() const {51    return used;52}53 54size_t common_ngram_mod::size() const {55    return entries.size();56}57 58size_t common_ngram_mod::size_bytes() const {59    return entries.size() * sizeof(entries[0]);60}61