KBaba7/llama.cpp

1// Unit tests for quantization specific functions - quantize, dequantize and dot product2 3#include "ggml.h"4#include "ggml-cpu.h"5 6#undef NDEBUG7#include <assert.h>8#include <math.h>9#include <stdio.h>10#include <string>11#include <vector>12 13#if defined(_MSC_VER)14#pragma warning(disable: 4244 4267) // possible loss of data15#endif16 17constexpr float MAX_QUANTIZATION_REFERENCE_ERROR = 0.0001f;18constexpr float MAX_QUANTIZATION_TOTAL_ERROR = 0.002f;19constexpr float MAX_QUANTIZATION_TOTAL_ERROR_TERNARY = 0.01f;20constexpr float MAX_QUANTIZATION_TOTAL_ERROR_2BITS = 0.0075f;21constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS = 0.0040f;22constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS = 0.0050f;23constexpr float MAX_DOT_PRODUCT_ERROR = 0.02f;24constexpr float MAX_DOT_PRODUCT_ERROR_LOWBIT = 0.04f;25constexpr float MAX_DOT_PRODUCT_ERROR_TERNARY = 0.15f;26 27static const char* RESULT_STR[] = {"ok", "FAILED"};28 29 30// Generate synthetic data31static void generate_data(float offset, size_t n, float * dst) {32    for (size_t i = 0; i < n; i++) {33        dst[i] = 0.1 + 2*cosf(i + offset);34    }35}36 37// Calculate RMSE between two float arrays38static float array_rmse(const float * a1, const float * a2, size_t n) {39    double sum = 0;40    for (size_t i = 0; i < n; i++) {41        double diff = a1[i] - a2[i];42        sum += diff * diff;43    }44    return sqrtf(sum) / n;45}46 47// Total quantization error on test data48static float total_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {49    std::vector<uint8_t> tmp_q(2*test_size);50    std::vector<float> tmp_out(test_size);51 52    qfns_cpu->from_float(test_data, tmp_q.data(), test_size);53    qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);54    return array_rmse(test_data, tmp_out.data(), test_size);55}56 57// Total quantization error on test data58static float reference_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {59    std::vector<uint8_t> tmp_q(2*test_size);60    std::vector<float> tmp_out(test_size);61    std::vector<float> tmp_out_ref(test_size);62 63    // FIXME: why is done twice?64    qfns_cpu->from_float(test_data, tmp_q.data(), test_size);65    qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);66 67    qfns->from_float_ref(test_data, tmp_q.data(), test_size);68    qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size);69 70    return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size);71}72 73static float dot_product(const float * a1, const float * a2, size_t test_size) {74    double sum = 0;75    for (size_t i = 0; i < test_size; i++) {76        sum += a1[i] * a2[i];77    }78    return sum;79}80 81// Total dot product error82static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) {83    GGML_UNUSED(qfns);84 85    std::vector<uint8_t> tmp_q1(2*test_size);86    std::vector<uint8_t> tmp_q2(2*test_size);87 88    const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);89 90    qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size);91    vdot->from_float(test_data2, tmp_q2.data(), test_size);92 93    float result = INFINITY;94    qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1);95 96    const float dot_ref = dot_product(test_data1, test_data2, test_size);97 98    return fabsf(result - dot_ref) / test_size;99}100 101int main(int argc, char * argv[]) {102    bool verbose = false;103    const size_t test_size = 32 * 128;104 105    std::string arg;106    for (int i = 1; i < argc; i++) {107        arg = argv[i];108 109        if (arg == "-v") {110            verbose = true;111        } else {112            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());113            return 1;114        }115    }116 117    std::vector<float> test_data(test_size);118    std::vector<float> test_data2(test_size);119 120    generate_data(0.0, test_data.size(), test_data.data());121    generate_data(1.0, test_data2.size(), test_data2.data());122 123    // Initialize GGML, ensures float conversion tables are initialized124    struct ggml_init_params ggml_params = {125        /* .mem_size   = */ 1*1024,126        /* .mem_buffer = */ NULL,127        /* .no_alloc   = */ true,128    };129    struct ggml_context * ctx = ggml_init(ggml_params);130 131    int num_failed = 0;132    bool failed = false;133 134    for (int i = 0; i < GGML_TYPE_COUNT; i++) {135        ggml_type type = (ggml_type) i;136        const auto * qfns = ggml_get_type_traits(type);137        const auto * qfns_cpu = ggml_get_type_traits_cpu(type);138 139        // deprecated - skip140        if (qfns->blck_size == 0) {141            continue;142        }143 144        const ggml_type ei = (ggml_type)i;145 146        printf("Testing %s\n", ggml_type_name((ggml_type) i));147        ggml_quantize_init(ei);148 149        if (qfns_cpu->from_float && qfns->to_float) {150            const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());151            const float max_quantization_error =152                type == GGML_TYPE_TQ1_0   ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :153                type == GGML_TYPE_TQ2_0   ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :154                type == GGML_TYPE_Q2_K    ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :155                type == GGML_TYPE_IQ2_S   ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :156                type == GGML_TYPE_Q3_K    ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :157                type == GGML_TYPE_IQ3_S   ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :158                type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS : MAX_QUANTIZATION_TOTAL_ERROR;159            failed = !(total_error < max_quantization_error);160            num_failed += failed;161            if (failed || verbose) {162                printf("%5s absolute quantization error:    %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);163            }164 165            const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());166            failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);167            num_failed += failed;168            if (failed || verbose) {169                printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);170            }171 172            const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data());173            const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||174                                            type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S175                                          ? MAX_DOT_PRODUCT_ERROR_LOWBIT176                                          : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0177                                          ? MAX_DOT_PRODUCT_ERROR_TERNARY178                                          : MAX_DOT_PRODUCT_ERROR;179            failed = !(vec_dot_error < max_allowed_error);180            num_failed += failed;181            if (failed || verbose) {182                printf("%5s dot product error:              %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);183            }184        }185    }186 187    if (num_failed || verbose) {188        printf("%d tests failed\n", num_failed);189    }190 191    ggml_free(ctx);192 193    return num_failed > 0;194}195