echodict/llama.cpp

1#include "hf-cache.h"2 3#include "build-info.h"4#include "common.h"5#include "log.h"6#include "http.h"7 8#define JSON_ASSERT GGML_ASSERT9#include <nlohmann/json.hpp>10 11#include <filesystem>12#include <fstream>13#include <atomic>14#include <regex> // migration only15#include <string>16#include <string_view>17#include <stdexcept>18 19namespace nl = nlohmann;20 21#if defined(_WIN32)22#define WIN32_LEAN_AND_MEAN23#ifndef NOMINMAX24#define NOMINMAX25#endif26#define HOME_DIR "USERPROFILE"27#include <windows.h>28#else29#define HOME_DIR "HOME"30#include <unistd.h>31#include <pwd.h>32#endif33 34namespace hf_cache {35 36namespace fs = std::filesystem;37 38static fs::path get_cache_directory() {39    static const fs::path cache = []() {40        struct {41            const char * var;42            fs::path path;43        } entries[] = {44            {"LLAMA_CACHE",           fs::path()},45            {"HF_HUB_CACHE",          fs::path()},46            {"HUGGINGFACE_HUB_CACHE", fs::path()},47            {"HF_HOME",               fs::path("hub")},48            {"XDG_CACHE_HOME",        fs::path("huggingface") / "hub"},49            {HOME_DIR,                fs::path(".cache") / "huggingface" / "hub"}50        };51        for (const auto & entry : entries) {52            if (auto * p = std::getenv(entry.var); p && *p) {53                fs::path base(p);54                return entry.path.empty() ? base : base / entry.path;55            }56        }57#ifndef _WIN3258        const struct passwd * pw = getpwuid(getuid());59 60        if (pw->pw_dir && *pw->pw_dir) {61            return fs::path(pw->pw_dir) / ".cache" / "huggingface" / "hub";62        }63#endif64        throw std::runtime_error("Failed to determine HF cache directory");65    }();66 67    return cache;68}69 70static std::string folder_name_to_repo(const std::string & folder) {71    constexpr std::string_view prefix = "models--";72    if (folder.rfind(prefix, 0)) {73        return {};74    }75    std::string result = folder.substr(prefix.length());76    string_replace_all(result, "--", "/");77    return result;78}79 80static std::string repo_to_folder_name(const std::string & repo_id) {81    constexpr std::string_view prefix = "models--";82    std::string result = std::string(prefix) + repo_id;83    string_replace_all(result, "/", "--");84    return result;85}86 87static fs::path get_repo_path(const std::string & repo_id) {88    return get_cache_directory() / repo_to_folder_name(repo_id);89}90 91static bool is_hex_char(const char c) {92    return (c >= 'A' && c <= 'F') ||93           (c >= 'a' && c <= 'f') ||94           (c >= '0' && c <= '9');95}96 97static bool is_hex_string(const std::string & s, size_t expected_len) {98    if (s.length() != expected_len) {99        return false;100    }101    for (const char c : s) {102        if (!is_hex_char(c)) {103            return false;104        }105    }106    return true;107}108 109static bool is_alphanum(const char c) {110    return (c >= 'A' && c <= 'Z') ||111           (c >= 'a' && c <= 'z') ||112           (c >= '0' && c <= '9');113}114 115static bool is_special_char(char c) {116    return c == '/' || c == '.' || c == '-';117}118 119// base chars [A-Za-z0-9_] are always valid120// special chars [/.-] must be surrounded by base chars121// exactly one '/' required122static bool is_valid_repo_id(const std::string & repo_id) {123    if (repo_id.empty() || repo_id.length() > 256) {124        return false;125    }126    int slash = 0;127    bool special = true;128 129    for (const char c : repo_id) {130        if (is_alphanum(c) || c == '_') {131            special = false;132        } else if (is_special_char(c)) {133            if (special) {134                return false;135            }136            slash += (c == '/');137            special = true;138        } else {139            return false;140        }141    }142    return !special && slash == 1;143}144 145static bool is_valid_hf_token(const std::string & token) {146    if (token.length() < 37 || token.length() > 256 ||147        !string_starts_with(token, "hf_")) {148        return false;149    }150    for (size_t i = 3; i < token.length(); ++i) {151        if (!is_alphanum(token[i])) {152            return false;153        }154    }155    return true;156}157 158static bool is_valid_commit(const std::string & hash) {159    return is_hex_string(hash, 40);160}161 162static bool is_valid_oid(const std::string & oid) {163    return is_hex_string(oid, 40) || is_hex_string(oid, 64);164}165 166static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) {167    if (subpath.is_absolute()) {168        return false; // never do a / b with b absolute169    }170    auto b = fs::absolute(path).lexically_normal();171    auto t = (b / subpath).lexically_normal();172    auto [b_end, _] = std::mismatch(b.begin(), b.end(), t.begin(), t.end());173 174    return b_end == b.end();175}176 177static void safe_write_file(const fs::path & path, const std::string & data) {178    fs::path path_tmp = path.string() + ".tmp";179 180    if (path.has_parent_path()) {181        fs::create_directories(path.parent_path());182    }183 184    std::ofstream file(path_tmp);185    file << data;186    file.close();187 188    std::error_code ec;189 190    if (!file.fail()) {191        fs::rename(path_tmp, path, ec);192    }193    if (file.fail() || ec) {194        fs::remove(path_tmp, ec);195        throw std::runtime_error("failed to write file: " + path.string());196    }197}198 199static nl::json api_get(const std::string & url,200                        const std::string & token) {201    auto [cli, parts] = common_http_client(url);202 203    httplib::Headers headers = {204        {"User-Agent", "llama-cpp/" + std::string(llama_build_info())},205        {"Accept", "application/json"}206    };207 208    if (is_valid_hf_token(token)) {209        headers.emplace("Authorization", "Bearer " + token);210    } else if (!token.empty()) {211        LOG_WRN("%s: invalid token, authentication disabled\n", __func__);212    }213 214    if (auto res = cli.Get(parts.path, headers)) {215        auto body = res->body;216 217        if (res->status == 200) {218            return nl::json::parse(res->body);219        }220        try {221            body = nl::json::parse(res->body)["error"].get<std::string>();222        } catch (...) { }223 224        throw std::runtime_error("GET failed (" + std::to_string(res->status) + "): " + body);225    } else {226        throw std::runtime_error("HTTPLIB failed: " + httplib::to_string(res.error()));227    }228}229 230static std::string get_repo_commit(const std::string & repo_id,231                                   const std::string & token) {232    try {233        auto endpoint = common_get_model_endpoint();234        auto json = api_get(endpoint + "api/models/" + repo_id + "/refs", token);235 236        if (!json.is_object() ||237            !json.contains("branches") || !json["branches"].is_array()) {238            LOG_WRN("%s: missing 'branches' for '%s'\n", __func__, repo_id.c_str());239            return {};240        }241 242        fs::path refs_path = get_repo_path(repo_id) / "refs";243        std::string name;244        std::string commit;245 246        for (const auto & branch : json["branches"]) {247            if (!branch.is_object() ||248                !branch.contains("name") || !branch["name"].is_string() ||249                !branch.contains("targetCommit") || !branch["targetCommit"].is_string()) {250                continue;251            }252            std::string _name = branch["name"].get<std::string>();253            std::string _commit = branch["targetCommit"].get<std::string>();254 255            if (!is_valid_subpath(refs_path, _name)) {256                LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());257                continue;258            }259            if (!is_valid_commit(_commit)) {260                LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());261                continue;262            }263 264            if (_name == "main") {265                name = _name;266                commit = _commit;267                break;268            }269 270            if (name.empty() || commit.empty()) {271                name = _name;272                commit = _commit;273            }274        }275 276        if (name.empty() || commit.empty()) {277            LOG_WRN("%s: no valid branch for '%s'\n", __func__, repo_id.c_str());278            return {};279        }280 281        safe_write_file(refs_path / name, commit);282        return commit;283 284    } catch (const nl::json::exception & e) {285        LOG_ERR("%s: JSON error: %s\n", __func__, e.what());286    } catch (const std::exception & e) {287        LOG_ERR("%s: error: %s\n", __func__, e.what());288    }289    return {};290}291 292hf_files get_repo_files(const std::string & repo_id,293                        const std::string & token) {294    if (!is_valid_repo_id(repo_id)) {295        LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str());296        return {};297    }298 299    std::string commit = get_repo_commit(repo_id, token);300    if (commit.empty()) {301        LOG_WRN("%s: failed to resolve commit for %s\n", __func__, repo_id.c_str());302        return {};303    }304 305    fs::path blobs_path = get_repo_path(repo_id) / "blobs";306    fs::path commit_path = get_repo_path(repo_id) / "snapshots" / commit;307 308    hf_files files;309 310    try {311        auto endpoint = common_get_model_endpoint();312        auto json = api_get(endpoint + "api/models/" + repo_id + "/tree/" + commit + "?recursive=true", token);313 314        if (!json.is_array()) {315            LOG_WRN("%s: response is not an array for '%s'\n", __func__, repo_id.c_str());316            return {};317        }318 319        for (const auto & item : json) {320            if (!item.is_object() ||321                !item.contains("type") || !item["type"].is_string() || item["type"] != "file" ||322                !item.contains("path") || !item["path"].is_string()) {323                continue;324            }325 326            hf_file file;327            file.repo_id = repo_id;328            file.path = item["path"].get<std::string>();329 330            if (!is_valid_subpath(commit_path, file.path)) {331                LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());332                continue;333            }334 335            if (item.contains("lfs") && item["lfs"].is_object()) {336                if (item["lfs"].contains("oid") && item["lfs"]["oid"].is_string()) {337                    file.oid = item["lfs"]["oid"].get<std::string>();338                }339                if (item["lfs"].contains("size") && item["lfs"]["size"].is_number()) {340                    file.size = item["lfs"]["size"].get<size_t>();341                }342            } else if (item.contains("oid") && item["oid"].is_string()) {343                file.oid = item["oid"].get<std::string>();344            }345            if (file.size == 0 && item.contains("size") && item["size"].is_number()) {346                file.size = item["size"].get<size_t>();347            }348 349            if (!file.oid.empty() && !is_valid_oid(file.oid)) {350                LOG_WRN("%s: skip invalid oid: %s\n", __func__, file.oid.c_str());351                continue;352            }353 354            file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;355 356            fs::path final_path = commit_path / file.path;357            file.final_path = final_path.string();358 359            if (!file.oid.empty() && !fs::exists(final_path)) {360                fs::path local_path = blobs_path / file.oid;361                file.local_path = local_path.string();362            } else {363                file.local_path = file.final_path;364            }365 366            files.push_back(file);367        }368    } catch (const nl::json::exception & e) {369        LOG_ERR("%s: JSON error: %s\n", __func__, e.what());370    } catch (const std::exception & e) {371        LOG_ERR("%s: error: %s\n", __func__, e.what());372    }373    return files;374}375 376static std::string get_cached_ref(const fs::path & repo_path) {377    fs::path refs_path = repo_path / "refs";378    if (!fs::is_directory(refs_path)) {379        return {};380    }381    std::string fallback;382 383    for (const auto & entry : fs::directory_iterator(refs_path)) {384        if (!entry.is_regular_file()) {385            continue;386        }387        std::ifstream f(entry.path());388        std::string commit;389        if (!f || !std::getline(f, commit) || commit.empty()) {390            continue;391        }392        if (!is_valid_commit(commit)) {393            LOG_WRN("%s: skip invalid commit: %s\n", __func__, commit.c_str());394            continue;395        }396        if (entry.path().filename() == "main") {397            return commit;398        }399        if (fallback.empty()) {400            fallback = commit;401        }402    }403    return fallback;404}405 406hf_files get_cached_files(const std::string & repo_id) {407    fs::path cache_dir = get_cache_directory();408    if (!fs::exists(cache_dir)) {409        return {};410    }411 412    if (!repo_id.empty() && !is_valid_repo_id(repo_id)) {413        LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str());414        return {};415    }416 417    hf_files files;418 419    for (const auto & repo : fs::directory_iterator(cache_dir)) {420        if (!repo.is_directory()) {421            continue;422        }423        fs::path snapshots_path = repo.path() / "snapshots";424 425        if (!fs::exists(snapshots_path)) {426            continue;427        }428        std::string _repo_id = folder_name_to_repo(repo.path().filename().string());429 430        if (!is_valid_repo_id(_repo_id)) {431            continue;432        }433        if (!repo_id.empty() && _repo_id != repo_id) {434            continue;435        }436        std::string commit = get_cached_ref(repo.path());437        fs::path commit_path = snapshots_path / commit;438 439        if (commit.empty() || !fs::is_directory(commit_path)) {440            continue;441        }442        for (const auto & entry : fs::recursive_directory_iterator(commit_path)) {443            if (!entry.is_regular_file() && !entry.is_symlink()) {444                continue;445            }446            fs::path path = entry.path().lexically_relative(commit_path);447 448            if (!path.empty()) {449                hf_file file;450                file.repo_id = _repo_id;451                file.path = path.generic_string();452                file.local_path = entry.path().string();453                file.final_path = file.local_path;454                files.push_back(std::move(file));455            }456        }457    }458 459    return files;460}461 462std::string finalize_file(const hf_file & file) {463    static std::atomic<bool> symlinks_disabled{false};464 465    std::error_code ec;466    fs::path local_path(file.local_path);467    fs::path final_path(file.final_path);468 469    if (local_path == final_path || fs::exists(final_path, ec)) {470        return file.final_path;471    }472 473    if (!fs::exists(local_path, ec)) {474        return file.final_path;475    }476 477    fs::create_directories(final_path.parent_path(), ec);478 479    if (!symlinks_disabled) {480        fs::path target = fs::relative(local_path, final_path.parent_path(), ec);481        if (!ec) {482            fs::create_symlink(target, final_path, ec);483        }484        if (!ec) {485            return file.final_path;486        }487    }488 489    if (!symlinks_disabled.exchange(true)) {490        LOG_WRN("%s: failed to create symlink: %s\n", __func__, ec.message().c_str());491        LOG_WRN("%s: switching to degraded mode\n", __func__);492    }493 494    fs::rename(local_path, final_path, ec);495    if (ec) {496        LOG_WRN("%s: failed to move file to snapshots: %s\n", __func__, ec.message().c_str());497        fs::copy(local_path, final_path, ec);498        if (ec) {499            LOG_ERR("%s: failed to copy file to snapshots: %s\n", __func__, ec.message().c_str());500        }501    }502    return file.final_path;503}504 505// delete everything after this line, one day506 507// copied from download.cpp without the tag part508struct gguf_split_info {509    std::string prefix; // tag included510    int index;511    int count;512};513 514static gguf_split_info get_gguf_split_info(const std::string & path) {515    static const std::regex re_split("^(.+)-([0-9]{5})-of-([0-9]{5})$", std::regex::icase);516    std::smatch m;517 518    std::string prefix = path;519    if (!string_remove_suffix(prefix, ".gguf")) {520        return {};521    }522 523    int index = 1;524    int count = 1;525 526    if (std::regex_match(prefix, m, re_split)) {527        index = std::stoi(m[2].str());528        count = std::stoi(m[3].str());529        prefix = m[1].str();530    }531 532    return {std::move(prefix), index, count};533}534 535static std::pair<std::string, std::string> parse_manifest_name(std::string & filename) {536    static const std::regex re(R"(^manifest=([^=]+)=([^=]+)=.*\.json$)");537    std::smatch match;538    if (std::regex_match(filename, match, re)) {539        return {match[1].str(), match[2].str()};540    }541    return {};542}543 544static std::string make_old_cache_filename(const std::string & owner,545                                           const std::string & repo,546                                           const std::string & filename) {547    auto result = owner + "_" + repo + "_" + filename;548    string_replace_all(result, "/", "_");549    return result;550}551 552struct migrate_file {553    std::string path;554    std::string sha256;555    size_t size;556    fs::path old_path;557    fs::path etag_path;558    const hf_file * file;559};560 561using migrate_files = std::vector<migrate_file>;562 563static bool collect_file(const fs::path    & old_cache,564                         const std::string & owner,565                         const std::string & repo,566                         const std::string & path,567                         const std::string & sha256,568                         const hf_files    & files,569                         migrate_files     & to_migrate) {570 571    const hf_file * file = nullptr;572 573    for (const auto & f : files) {574        if (f.path == path) {575            file = &f;576            break;577        }578    }579 580    std::string old_filename = make_old_cache_filename(owner, repo, path);581    fs::path old_path = old_cache / old_filename;582    fs::path etag_path = old_path.string() + ".etag";583 584    if (!fs::exists(old_path)) {585        if (file && fs::exists(file->final_path)) {586            return true;587        }588        LOG_WRN("%s: %s not found in old cache or HF cache\n", __func__, old_filename.c_str());589        return false;590    }591 592    if (!file) {593        LOG_WRN("%s: %s not found in current repo\n", __func__, old_filename.c_str());594        return false;595    }596 597    if (!sha256.empty() && !file->oid.empty() && sha256 != file->oid) {598        LOG_WRN("%s: %s is not up to date (sha256 mismatch)\n", __func__, old_filename.c_str());599        return false;600    }601 602    if (file->size > 0) {603        size_t size = fs::file_size(old_path);604        if (size != file->size) {605            LOG_WRN("%s: %s has wrong size %zu (expected %zu)\n", __func__, old_filename.c_str(), size, file->size);606            return false;607        }608    }609 610    to_migrate.push_back({path, sha256, file->size, old_path, etag_path, file});611    return true;612}613 614static bool collect_files(const fs::path    & old_cache,615                          const std::string & owner,616                          const std::string & repo,617                          const nl::json    & node,618                          const hf_files    & files,619                          migrate_files     & to_migrate) {620 621    if (!node.contains("rfilename") ||622        !node.contains("lfs")       ||623        !node["lfs"].contains("sha256")) {624        return true;625    }626 627    std::string path = node["rfilename"];628    std::string sha256 = node["lfs"]["sha256"];629 630    auto split = get_gguf_split_info(path);631 632    if (split.count <= 1) {633        return collect_file(old_cache, owner, repo, path, sha256, files, to_migrate);634    }635 636    std::vector<std::pair<std::string, std::string>> splits;637 638    for (const auto & f : files) {639        auto split_f = get_gguf_split_info(f.path);640        if (split_f.count == split.count && split_f.prefix == split.prefix) {641            // sadly the manifest only provides the sha256 of the first file (index == 1)642            // the rest will be verified using the size...643            std::string f_sha256 = (split_f.index == 1) ? sha256 : "";644            splits.emplace_back(f.path, f_sha256);645        }646    }647 648    if ((int)splits.size() != split.count) {649        LOG_WRN("%s: expected %d split files but found %d in repo\n", __func__, split.count, (int)splits.size());650        return false;651    }652 653    for (const auto & [f_path, f_sha256] : splits) {654        if (!collect_file(old_cache, owner, repo, f_path, f_sha256, files, to_migrate)) {655            return false;656        }657    }658 659    return true;660}661 662static bool migrate_file(const migrate_file & file) {663    std::error_code ec;664 665    fs::path new_path(file.file->local_path);666    fs::create_directories(new_path.parent_path(), ec);667 668    if (!fs::exists(new_path, ec)) {669        fs::rename(file.old_path, new_path, ec);670        if (ec) {671            fs::copy_file(file.old_path, new_path, ec);672            if (ec) {673                LOG_ERR("%s: failed to move/copy %s: %s\n", __func__, file.old_path.string().c_str(), ec.message().c_str());674                return false;675            }676        }677        fs::remove(file.old_path, ec);678    }679    fs::remove(file.etag_path, ec);680 681    std::string filename = finalize_file(*file.file);682    LOG_INF("%s: migrated %s -> %s\n", __func__, file.old_path.filename().string().c_str(), filename.c_str());683    return true;684}685 686void migrate_old_cache_to_hf_cache(const std::string & token, bool offline) {687    fs::path old_cache = fs_get_cache_directory();688    if (!fs::exists(old_cache)) {689        return;690    }691 692    if (offline) {693        LOG_WRN("%s: skipping migration in offline mode (will run when online)\n", __func__);694        return; // -hf is not going to work695    }696 697    bool warned = false;698 699    for (const auto & entry : fs::directory_iterator(old_cache)) {700        if (!entry.is_regular_file()) {701            continue;702        }703        auto filename = entry.path().filename().string();704        auto [owner, repo] = parse_manifest_name(filename);705 706        if (owner.empty() || repo.empty()) {707            continue;708        }709 710        if (!warned) {711            warned = true;712            LOG_WRN("================================================================================\n"713                    "WARNING: Migrating cache to HuggingFace cache directory\n"714                    "  Old cache: %s\n"715                    "  New cache: %s\n"716                    "This one-time migration moves models previously downloaded with -hf\n"717                    "from the legacy llama.cpp cache to the standard HuggingFace cache.\n"718                    "Models downloaded with --model-url are not affected.\n"719                    "================================================================================\n",720                    old_cache.string().c_str(), get_cache_directory().string().c_str());721        }722 723        auto repo_id = owner + "/" + repo;724        auto files = get_repo_files(repo_id, token);725 726        if (files.empty()) {727            LOG_WRN("%s: could not get repo files for %s, skipping\n", __func__, repo_id.c_str());728            continue;729        }730 731        migrate_files to_migrate;732        bool ok = true;733 734        try {735            std::ifstream manifest(entry.path());736            auto json = nl::json::parse(manifest);737            for (const char * key : {"ggufFile", "mmprojFile"}) {738                if (json.contains(key)) {739                    if (!collect_files(old_cache, owner, repo, json[key], files, to_migrate)) {740                        ok = false;741                        break;742                    }743                }744            }745        } catch (const std::exception & e) {746            LOG_WRN("%s: failed to parse manifest %s: %s\n", __func__, filename.c_str(), e.what());747            continue;748        }749 750        if (!ok) {751            LOG_WRN("%s: migration skipped: one or more files failed validation\n", __func__);752            continue;753        }754 755        for (const auto & file : to_migrate) {756            if (!migrate_file(file)) {757                ok = false;758                break;759            }760        }761 762        if (!ok) {763            LOG_WRN("%s: migration failed: could not migrate all files\n", __func__);764            continue;765        }766 767        LOG_INF("%s: migration complete, deleting manifest: %s\n", __func__, entry.path().string().c_str());768        fs::remove(entry.path());769    }770}771 772} // namespace hf_cache773