Brunobkr/llama.cpp_AlgMor24_github

1// llama-ui-embed: generate ui.cpp / ui.h that embed UI assets as C arrays.2//3// Usage:4//   llama-ui-embed <out_cpp> <out_h> [<asset_dir>]5//6// Recursively embeds every regular file under <asset_dir>.7// Asset names are relative paths from <asset_dir> (e.g. "_app/immutable/bundle.HASH.js").8// Without <asset_dir>, emits an empty asset table.9 10#include <inttypes.h>11#include <stdarg.h>12#include <stdint.h>13#include <stdio.h>14#include <string.h>15 16#include <algorithm>17#include <filesystem>18#include <fstream>19#include <functional>20#include <string>21#include <vector>22 23 24static const char * mime_from_ext(const std::string & name) {25    auto ext = name.rfind('.');26    if (ext == std::string::npos) return "application/octet-stream";27    std::string e = name.substr(ext + 1);28    if (e == "html")        return "text/html; charset=utf-8";29    if (e == "css")         return "text/css";30    if (e == "js")          return "application/javascript";31    if (e == "json")        return "application/json";32    if (e == "webmanifest") return "application/manifest+json";33    if (e == "svg")         return "image/svg+xml";34    if (e == "png")         return "image/png";35    if (e == "jpg" ||36        e == "jpeg")        return "image/jpeg";37    if (e == "ico")         return "image/x-icon";38    if (e == "woff")        return "font/woff";39    if (e == "woff2")       return "font/woff2";40    return "application/octet-stream";41}42 43// Computes FNV-1a hash of the data44static uint64_t fnv_hash(const uint8_t * data, size_t len) {45    const uint64_t fnv_prime = 0x100000001b3ULL;46    uint64_t hash = 0xcbf29ce484222325ULL;47 48    for (size_t i = 0; i < len; ++i) {49        hash ^= data[i];50        hash *= fnv_prime;51    }52    return hash;53}54 55static bool read_file(const std::filesystem::path & path, std::vector<unsigned char> & out) {56    std::ifstream f(path, std::ios::binary | std::ios::ate);57    if (!f) {58        fprintf(stderr, "embed: cannot open %s\n", path.string().c_str());59        return false;60    }61    const auto sz = f.tellg();62    if (sz < 0) {63        return false;64    }65    f.seekg(0);66    out.resize(static_cast<size_t>(sz));67    if (sz > 0 && !f.read(reinterpret_cast<char *>(out.data()), sz)) {68        return false;69    }70    return true;71}72 73static void append_bytes_hex(std::string & out, const std::vector<unsigned char> & bytes) {74    static const char hex[] = "0123456789abcdef";75    out.reserve(out.size() + bytes.size() * 5);76    for (unsigned char b : bytes) {77        out += '0';78        out += 'x';79        out += hex[b >> 4];80        out += hex[b & 0xf];81        out += ',';82    }83}84 85static bool write_if_different(const std::string & path, const std::string & content) {86    std::ifstream f(path, std::ios::binary | std::ios::ate);87    if (f) {88        const auto sz = f.tellg();89        if (sz >= 0 && static_cast<size_t>(sz) == content.size()) {90            std::string existing(static_cast<size_t>(sz), '\0');91            f.seekg(0);92            if (sz == 0 || f.read(existing.data(), sz)) {93                if (existing == content) {94                    return true;95                }96            }97        }98    }99 100    std::ofstream out(path, std::ios::binary | std::ios::trunc);101    if (!out) {102        fprintf(stderr, "embed: cannot write %s\n", path.c_str());103        return false;104    }105    if (!content.empty()) {106        out.write(content.data(), static_cast<std::streamsize>(content.size()));107    }108    bool ok = out.good();109    if (ok) {110        printf("embed: write output file %s\n", path.c_str());111    }112    return ok;113}114 115static std::string path_basename(const std::string & name) {116    const size_t p = name.rfind('/');117    return p == std::string::npos ? name : name.substr(p + 1);118}119static bool str_starts_with(const std::string & s, const char * prefix) {120    const size_t n = strlen(prefix);121    return s.size() >= n && s.compare(0, n, prefix) == 0;122}123static bool str_ends_with(const std::string & s, const char * suffix) {124    const size_t n = strlen(suffix);125    return s.size() >= n && s.compare(s.size() - n, n, suffix) == 0;126}127 128static std::string fmt(const char * pattern, ...) {129    char tmp[512];130    va_list ap;131    va_start(ap, pattern);132    const int n = vsnprintf(tmp, sizeof(tmp), pattern, ap);133    va_end(ap);134    return (n > 0) ? std::string(tmp, static_cast<size_t>(n)) : std::string();135}136 137struct asset_entry {138    std::string           name;139    std::filesystem::path path;140};141 142int main(int argc, char ** argv) {143    if (argc < 3 || argc > 4) {144        fprintf(stderr, "usage: %s <out_cpp> <out_h> [<asset_dir>]\n", argv[0]);145        return 1;146    }147 148    const std::string out_cpp   = argv[1];149    const std::string out_h     = argv[2];150    const std::string asset_dir = (argc >= 4) ? argv[3] : std::string();151 152    const bool        use_gzip = !asset_dir.empty() && std::filesystem::exists(asset_dir + "/_gzip");153    const std::string in_dir   = use_gzip ? (asset_dir + "/_gzip") : asset_dir;154 155    std::vector<asset_entry> assets;156    if (!in_dir.empty()) {157        const std::filesystem::path dir = in_dir;158 159        std::error_code ec;160        std::filesystem::recursive_directory_iterator it(dir, ec);161        if (ec) {162            fprintf(stderr, "embed: cannot iterate %s: %s\n", argv[3], ec.message().c_str());163            return 1;164        }165        for (const auto & entry : it) {166            if (!entry.is_regular_file()) {167                continue;168            }169            // name is the relative path from dir, with forward slashes170            const std::string name = entry.path().lexically_relative(dir).generic_string();171            assets.push_back({ name, entry.path() });172        }173 174        // directory iteration order is unspecified; sort for reproducible output175        std::sort(assets.begin(), assets.end(),176                  [](const asset_entry & a, const asset_entry & b) { return a.name < b.name; });177    }178 179    const int n_assets = static_cast<int>(assets.size());180 181    if (n_assets > 0) {182        using match_fn = std::function<bool(const std::string &)>;183        auto exact = [](const char * name) -> match_fn {184            return [name](const std::string & base) { return base == name; };185        };186 187        struct required_check { const char * label; match_fn match; bool found; };188        required_check checks[] = {189            { "index.html",           exact("index.html"),           false },190            { "manifest.webmanifest", exact("manifest.webmanifest"), false },191            { "sw.js",                exact("sw.js"),                false },192            { "build.json",           exact("build.json"),           false },193            { "version.json",         exact("version.json"),         false },194            { "bundle[hash].js",      [](const std::string & b) {195                return str_starts_with(b, "bundle")  && str_ends_with(b, ".js");196            }, false },197            { "bundle[hash].css",     [](const std::string & b) {198                return str_starts_with(b, "bundle")  && str_ends_with(b, ".css");199            }, false },200            { "workbox[hash].js",     [](const std::string & b) {201                return str_starts_with(b, "workbox") && str_ends_with(b, ".js");202            }, false },203        };204 205        for (const auto & a : assets) {206            const std::string base = path_basename(a.name);207            for (auto & c : checks) {208                if (!c.found) { c.found = c.match(base); }209            }210        }211 212        std::vector<const char *> missing;213        for (const auto & c : checks) {214            if (!c.found) { missing.push_back(c.label); }215        }216        if (!missing.empty()) {217            fprintf(stderr, "\ncurrent asset files:\n");218            for (const auto & a : assets) {219                fprintf(stderr, "    %s\n", a.name.c_str());220            }221            fprintf(stderr, "missing required asset(s):\n");222            for (const char * m : missing) {223                fprintf(stderr, "    %s\n", m);224            }225            fprintf(stderr, "hint: try cleaning your build directory: %s\n", in_dir.c_str());226            return 1;227        }228    }229 230    std::string h;231    h += "#pragma once\n\n#include <array>\n#include <string>\n\n";232    if (n_assets > 0) {233        h += "#define LLAMA_UI_HAS_ASSETS 1\n\n";234    }235    h +=236        "struct llama_ui_asset {\n"237        "    std::string           name;\n"238        "    const unsigned char * data;\n"239        "    std::size_t           size;\n"240        "    std::string           etag;\n"241        "    std::string           type;\n"242        "};\n\n"243        "const llama_ui_asset * llama_ui_find_asset(const std::string & name);\n"244        "bool llama_ui_use_gzip();\n";245    h += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets();\n", n_assets);246 247    std::string cpp;248    cpp += "#include \"ui.h\"\n\n";249 250    if (n_assets > 0) {251        for (int i = 0; i < n_assets; i++) {252            std::vector<unsigned char> bytes;253            if (!read_file(assets[i].path, bytes)) {254                return 1;255            }256            if (bytes.empty()) {257                fprintf(stderr, "embed: empty file: %s\n", assets[i].path.generic_string().c_str());258                return 1;259            }260            cpp += fmt("static const unsigned char asset_%d_data[] = {", i);261            append_bytes_hex(cpp, bytes);262            const auto hash = fnv_hash(bytes.data(), bytes.size());263 264            cpp += fmt("};\nstatic const std::size_t   asset_%d_size = %zu;\n",265                       i, bytes.size());266            cpp += fmt("static const char          asset_%d_etag[] = \"\\\"0x%016" PRIx64 "\\\"\";\n\n",267                       i, hash);268        }269 270        cpp += fmt("static const std::array<llama_ui_asset, %d> g_assets = {{\n", n_assets);271        for (int i = 0; i < n_assets; i++) {272            const std::string & name = assets[i].name;273            cpp += fmt("    { \"%s\", asset_%d_data, asset_%d_size, asset_%d_etag, \"%s\" },\n",274                       name.c_str(), i, i, i, mime_from_ext(name));275        }276        cpp += "}};\n\n";277 278        cpp +=279            "const llama_ui_asset * llama_ui_find_asset(const std::string & name) {\n"280            "    for (const auto & a : g_assets) {\n"281            "        if (a.name == name) {\n"282            "            return &a;\n"283            "        }\n"284            "    }\n"285            "    return nullptr;\n"286            "}\n";287        cpp += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets() {\n", n_assets);288        cpp +=     "    return g_assets;\n"289                   "}\n";290    } else {291        cpp +=292            "const llama_ui_asset * llama_ui_find_asset(const std::string &) {\n"293            "    return nullptr;\n"294            "}\n"295            "const std::array<llama_ui_asset, 0> & llama_ui_get_assets() {\n"296            "    static const std::array<llama_ui_asset, 0> empty{};\n"297            "    return empty;\n"298            "}\n";299    }300    cpp += fmt("bool llama_ui_use_gzip() { return %s; }\n", use_gzip ? "true" : "false");301 302    bool ok = true;303    ok = write_if_different(out_h,   h)   && ok;304    ok = write_if_different(out_cpp, cpp) && ok;305    return ok ? 0 : 1;306}307