echodict/llama.cpp

1#pragma once2 3#include <cstdint>4#include <vector>5#include <unordered_map>6#include <unordered_set>7 8struct range_nfd {9    uint32_t first;10    uint32_t last;11    uint32_t nfd;12};13 14static const uint32_t MAX_CODEPOINTS = 0x110000;15 16extern const std::initializer_list<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;17extern const std::unordered_set<uint32_t> unicode_set_whitespace;18extern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase;19extern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_uppercase;20extern const std::initializer_list<range_nfd> unicode_ranges_nfd;21