Brunobkr/llama.cpp_AlgMor24_github

1// Unit tests for quantization specific functions - quantize, dequantize and dot product2 3#include "ggml.h"4#include "ggml-cpu.h"5 6#undef NDEBUG7#include <assert.h>8#include <math.h>9#include <stdio.h>10#include <string>11#include <vector>12 13#if defined(_MSC_VER)14#pragma warning(disable: 4244 4267) // possible loss of data15#endif16 17constexpr float MAX_QUANTIZATION_REFERENCE_ERROR = 0.0001f;18constexpr float MAX_QUANTIZATION_TOTAL_ERROR = 0.002f;19constexpr float MAX_QUANTIZATION_TOTAL_ERROR_BINARY = 0.025f;20constexpr float MAX_QUANTIZATION_TOTAL_ERROR_TERNARY = 0.01f;21constexpr float MAX_QUANTIZATION_TOTAL_ERROR_2BITS = 0.0075f;22constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS = 0.0040f;23constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS = 0.0050f;24constexpr float MAX_QUANTIZATION_TOTAL_ERROR_FP4 = 0.0030f;25constexpr float MAX_DOT_PRODUCT_ERROR = 0.02f;26constexpr float MAX_DOT_PRODUCT_ERROR_LOWBIT = 0.04f;27constexpr float MAX_DOT_PRODUCT_ERROR_FP4 = 0.03f;28constexpr float MAX_DOT_PRODUCT_ERROR_BINARY = 0.40f;29constexpr float MAX_DOT_PRODUCT_ERROR_TERNARY = 0.15f;30 31static const char* RESULT_STR[] = {"ok", "FAILED"};32 33 34// Generate synthetic data35static void generate_data(float offset, size_t n, float * dst) {36    for (size_t i = 0; i < n; i++) {37        dst[i] = 0.1 + 2*cosf(i + offset);38    }39}40 41// Calculate RMSE between two float arrays42static float array_rmse(const float * a1, const float * a2, size_t n) {43    double sum = 0;44    for (size_t i = 0; i < n; i++) {45        double diff = a1[i] - a2[i];46        sum += diff * diff;47    }48    return sqrtf(sum) / n;49}50 51// Total quantization error on test data52static float total_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {53    std::vector<uint8_t> tmp_q(2*test_size);54    std::vector<float> tmp_out(test_size);55 56    qfns_cpu->from_float(test_data, tmp_q.data(), test_size);57    qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);58    return array_rmse(test_data, tmp_out.data(), test_size);59}60 61// Total quantization error on test data62static float reference_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {63    std::vector<uint8_t> tmp_q(2*test_size);64    std::vector<float> tmp_out(test_size);65    std::vector<float> tmp_out_ref(test_size);66 67    // FIXME: why is done twice?68    qfns_cpu->from_float(test_data, tmp_q.data(), test_size);69    qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);70 71    qfns->from_float_ref(test_data, tmp_q.data(), test_size);72    qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size);73 74    return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size);75}76 77static float dot_product(const float * a1, const float * a2, size_t test_size) {78    double sum = 0;79    for (size_t i = 0; i < test_size; i++) {80        sum += a1[i] * a2[i];81    }82    return sum;83}84 85// Total dot product error86static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) {87    GGML_UNUSED(qfns);88 89    std::vector<uint8_t> tmp_q1(2*test_size);90    std::vector<uint8_t> tmp_q2(2*test_size);91 92    const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);93 94    qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size);95    vdot->from_float(test_data2, tmp_q2.data(), test_size);96 97    float result = INFINITY;98    qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1);99 100    const float dot_ref = dot_product(test_data1, test_data2, test_size);101 102    return fabsf(result - dot_ref) / test_size;103}104 105static int test_vec_dot_f32(bool verbose) {106    const auto * f32 = ggml_get_type_traits_cpu(GGML_TYPE_F32);107    int num_failed = 0;108    for (int n : {1, 2, 3, 5, 7, 8, 15, 16, 17, 31, 33, 63, 67, 127, 129, 193, 255, 1023}) {109        std::vector<float> a(n);110        std::vector<float> b(n);111        generate_data(0.0, n, a.data());112        generate_data(1.0, n, b.data());113 114        float result = 0.0f;115        f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1);116        const float ref = dot_product(a.data(), b.data(), n);117        const float error = fabsf(result - ref) / n;118 119        const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR);120        num_failed += failed;121        if (failed || verbose) {122            printf(" f32 vec_dot n=%4d:                 %s (ref=%f got=%f err=%f)\n",123                   n, RESULT_STR[failed], ref, result, error);124        }125    }126    return num_failed;127}128 129static int test_vec_dot_q(bool verbose) {130    int num_failed = 0;131 132    const size_t test_size = 32 * 128;133 134    std::vector<float> test_data(test_size);135    std::vector<float> test_data2(test_size);136 137    generate_data(0.0, test_data.size(), test_data.data());138    generate_data(1.0, test_data2.size(), test_data2.data());139 140    for (int i = 0; i < GGML_TYPE_COUNT; i++) {141        ggml_type type = (ggml_type) i;142        const auto * qfns = ggml_get_type_traits(type);143        const auto * qfns_cpu = ggml_get_type_traits_cpu(type);144 145        // deprecated - skip146        if (qfns->blck_size == 0) {147            continue;148        }149 150        const ggml_type ei = (ggml_type)i;151 152        printf("Testing %s\n", ggml_type_name((ggml_type) i));153        ggml_quantize_init(ei);154 155        if (qfns_cpu->from_float && qfns->to_float) {156            const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());157            const float max_quantization_error =158                type == GGML_TYPE_Q1_0    ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :159                type == GGML_TYPE_TQ1_0   ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :160                type == GGML_TYPE_TQ2_0   ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :161                type == GGML_TYPE_Q2_0    ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :162                type == GGML_TYPE_Q2_K    ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :163                type == GGML_TYPE_IQ2_S   ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :164                type == GGML_TYPE_Q3_K    ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :165                type == GGML_TYPE_IQ3_S   ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :166                type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS :167                type == GGML_TYPE_NVFP4   ? MAX_QUANTIZATION_TOTAL_ERROR_FP4 : MAX_QUANTIZATION_TOTAL_ERROR;168            bool failed = !(total_error < max_quantization_error);169            num_failed += failed;170            if (failed || verbose) {171                printf("%5s absolute quantization error:    %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);172            }173 174            const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());175            failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);176            num_failed += failed;177            if (failed || verbose) {178                printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);179            }180 181            const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data());182            const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||183                type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S184                ? MAX_DOT_PRODUCT_ERROR_LOWBIT185                : type == GGML_TYPE_Q1_0186                ? MAX_DOT_PRODUCT_ERROR_BINARY187                : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0188                ? MAX_DOT_PRODUCT_ERROR_TERNARY189                : type == GGML_TYPE_NVFP4190                ? MAX_DOT_PRODUCT_ERROR_FP4191                : MAX_DOT_PRODUCT_ERROR;192            failed = !(vec_dot_error < max_allowed_error);193            num_failed += failed;194            if (failed || verbose) {195                printf("%5s dot product error:              %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);196            }197        }198    }199 200    return num_failed;201}202 203int main(int argc, char * argv[]) {204    bool verbose = false;205 206    std::string arg;207    for (int i = 1; i < argc; i++) {208        arg = argv[i];209 210        if (arg == "-v") {211            verbose = true;212        } else {213            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());214            return 1;215        }216    }217 218    ggml_cpu_init();219 220    int num_failed = 0;221 222    num_failed += test_vec_dot_f32(verbose);223    num_failed += test_vec_dot_q(verbose);224 225    if (num_failed || verbose) {226        printf("%d tests failed\n", num_failed);227    }228 229    return num_failed > 0;230}231