echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0479
1// This file defines tests for various GGML ops and backends.2// For the forward pass it asserts that the results of multiple backends computing the same GGML ops are consistent.3// For the backward pass it asserts that the gradients from backpropagation are consistent4// with the gradients obtained via the method of finite differences ("grad" mode, this is optional).5// It is also possible to check the performance ("perf" mode).6//7// this file has three sections: Section 1 does general setup, section 2 defines the GGML ops to be tested,8// and section 3 defines which tests to run.9// Quick start for adding a new GGML op: Go to section 2 and create a struct that inherits from test_case,10// then go to section 3 and add an instantiation of your struct.11 12 13// ##############################14// ## Section 1: General Setup ##15// ##############################16 17 18#include <ggml.h>19#include <ggml-alloc.h>20#include <ggml-backend.h>21#include <ggml-cpp.h>22 23#include <algorithm>24#include <array>25#include <cfloat>26#include <cinttypes>27#include <cstdarg>28#include <cstdint>29#include <cstdio>30#include <cstdlib>31#include <cstring>32#include <ctime>33#include <future>34#include <fstream>35#include <memory>36#include <random>37#include <regex>38#include <set>39#include <sstream>40#include <string>41#include <string_view>42#include <thread>43#include <vector>44#include <unordered_map>45 46#ifdef __EMSCRIPTEN__47# define N_THREADS 148#else49# define N_THREADS std::thread::hardware_concurrency()50#endif51 52static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {53 size_t nels = ggml_nelements(tensor);54 std::vector<float> data(nels);55 {56 // parallel initialization57 static const size_t n_threads = N_THREADS;58 // static RNG initialization (revisit if n_threads stops being constant)59 static std::vector<std::default_random_engine> generators = []() {60 std::random_device rd;61 std::vector<std::default_random_engine> vec;62 vec.reserve(n_threads);63 //for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(1234 + i); } // fixed seed64 for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(rd()); }65 return vec;66 }();67 68 auto init_thread = [&](size_t ith, size_t start, size_t end) {69 std::uniform_real_distribution<float> distribution(min, max);70 auto & gen = generators[ith];71 for (size_t i = start; i < end; i++) {72 data[i] = distribution(gen);73 }74 };75 76 if (n_threads == 1) {77 init_thread(0, 0, nels);78 } else {79 std::vector<std::future<void>> tasks;80 tasks.reserve(n_threads);81 for (size_t i = 0; i < n_threads; i++) {82 size_t start = i*nels/n_threads;83 size_t end = (i+1)*nels/n_threads;84 tasks.push_back(std::async(std::launch::async, init_thread, i, start, end));85 }86 for (auto & t : tasks) {87 t.get();88 }89 }90 }91 92 if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) {93 ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float));94 } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {95 GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0);96 97 // dummy importance matrix98 std::vector<float> imatrix(tensor->ne[0], 1.0f);99 const float * im = imatrix.data();100 if (!ggml_quantize_requires_imatrix(tensor->type)) {101 // when the imatrix is optional, we want to test both quantization with and without imatrix102 // use one of the random numbers to decide103 if (data[0] > 0.5f*(min + max)) {104 im = nullptr;105 }106 }107 108 std::vector<uint8_t> dataq(ggml_row_size(tensor->type, nels));109 {110 // parallel quantization by block111 size_t blck_size = ggml_blck_size(tensor->type);112 size_t n_blocks = nels / blck_size;113 114 auto quantize_thread = [&](size_t start, size_t end) {115 ggml_quantize_chunk(tensor->type, data.data(), dataq.data(),116 start * blck_size, end - start, blck_size, im);117 };118 119 const size_t min_blocks_per_thread = 1;120 const size_t n_quant_threads = std::min<size_t>(std::max<size_t>(N_THREADS/2, 1),121 std::max<size_t>(1, n_blocks / min_blocks_per_thread));122 123 if (n_quant_threads == 1) {124 // single-threaded quantization: do all blocks in the current thread125 quantize_thread(0, n_blocks);126 } else {127 std::vector<std::future<void>> tasks;128 tasks.reserve(n_quant_threads);129 for (size_t i = 0; i < n_quant_threads; i++) {130 size_t start = i*n_blocks/n_quant_threads;131 size_t end = (i+1)*n_blocks/n_quant_threads;132 tasks.push_back(std::async(std::launch::async, quantize_thread, start, end));133 }134 for (auto & t : tasks) {135 t.get();136 }137 }138 }139 ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size());140 } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) {141 // This is going to create some weird integers though.142 ggml_backend_tensor_set(tensor, data.data(), 0, ggml_nbytes(tensor));143 } else if (tensor->type == GGML_TYPE_I64) {144 // Integers with a size of 8 bytes can be set by mirroring the float data, the specific values are again not really meaningful.145 const size_t nbytes_half = ggml_nbytes(tensor)/2;146 ggml_backend_tensor_set(tensor, data.data(), 0*nbytes_half, nbytes_half);147 ggml_backend_tensor_set(tensor, data.data(), 1*nbytes_half, nbytes_half);148 } else {149 GGML_ABORT("fatal error");150 }151}152 153// generate an F16 mask where certain blocks are randomly masked with -INF value154static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {155 GGML_ASSERT(tensor->type == GGML_TYPE_F16);156 157 GGML_TENSOR_LOCALS( int32_t, ne, tensor, ne);158 159 std::vector<float> data_f32(ne0*ne1*ne2*ne3);160 std::vector<ggml_fp16_t> data_f16(ne0*ne1*ne2*ne3);161 162 std::random_device rd;163 std::mt19937 gen(rd());164 std::uniform_real_distribution<float> dis(min, max);165 166 for (size_t i = 0; i < data_f32.size(); i++) {167 data_f32[i] = dis(gen);168 }169 170 // block size171 const int blck0 = 128;172 const int blck1 = 64;173 174 // number of INF/zero blocks175 const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1);176 177 for (int b = 0; b < n_inf_zero_blocks; b++) {178 const int p3 = (rd() % ne3);179 const int p2 = (rd() % ne2);180 const int p1 = (rd() % ne1);181 const int p0 = (rd() % ne0);182 183 bool inf = rd() & 1;184 185 for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) {186 const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0;187 188 for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) {189 data_f32[idx + i0] = inf ? -INFINITY : 0.0f;190 }191 }192 }193 194 ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3);195 196 ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));197}198 199// generate a lower triangular matrix200static void init_tensor_tril(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {201 GGML_ASSERT(tensor->type == GGML_TYPE_F32);202 GGML_ASSERT(tensor->ne[0] == tensor->ne[1]);203 204 GGML_TENSOR_LOCALS(int32_t, ne, tensor, ne);205 GGML_TENSOR_LOCALS(size_t, nb, tensor, nb);206 207 std::vector<float> data_f32(ne0*ne1*ne2*ne3);208 209 std::random_device rd;210 std::mt19937 gen(rd());211 std::uniform_real_distribution<float> dis(min, max);212 213 for (int64_t i3 = 0; i3 < ne3; i3++) {214 for (int64_t i2 = 0; i2 < ne2; i2++) {215 for (int64_t i1 = 0; i1 < ne1; i1++) {216 for (int64_t i0 = 0; i0 < ne0; i0++) {217 int64_t idx = (i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3) / sizeof(float);218 if (i0 <= i1) {219 data_f32[idx] = dis(gen);220 } else {221 data_f32[idx] = 0.0f;222 }223 }224 }225 }226 }227 228 ggml_backend_tensor_set(tensor, data_f32.data(), 0, ggml_nbytes(tensor));229}230 231static std::vector<float> tensor_to_float(const ggml_tensor * t) {232 std::vector<float> tv;233 tv.reserve(ggml_nelements(t));234 235 std::vector<uint8_t> buf(ggml_nbytes(t));236 ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t));237 238 const auto * tt = ggml_get_type_traits(t->type);239 size_t bs = ggml_blck_size(t->type);240 std::vector<float> vq(ggml_blck_size(t->type));241 bool quantized = ggml_is_quantized(t->type);242 243 // access elements by index to avoid gaps in views244 for (int64_t i3 = 0; i3 < t->ne[3]; i3++) {245 for (int64_t i2 = 0; i2 < t->ne[2]; i2++) {246 for (int64_t i1 = 0; i1 < t->ne[1]; i1++) {247 for (int64_t i0 = 0; i0 < t->ne[0]; i0 += bs) {248 size_t i = i3*t->nb[3] + i2*t->nb[2] + i1*t->nb[1] + i0/bs*t->nb[0];249 if (t->type == GGML_TYPE_F16) {250 tv.push_back(ggml_fp16_to_fp32(*(ggml_fp16_t*)&buf[i]));251 } else if (t->type == GGML_TYPE_BF16) {252 tv.push_back(ggml_bf16_to_fp32(*(ggml_bf16_t*)&buf[i]));253 } else if (t->type == GGML_TYPE_F32) {254 tv.push_back(*(float *) &buf[i]);255 } else if (t->type == GGML_TYPE_I64) {256 tv.push_back((float)*(int64_t *) &buf[i]);257 } else if (t->type == GGML_TYPE_I32) {258 tv.push_back((float)*(int32_t *) &buf[i]);259 } else if (t->type == GGML_TYPE_I16) {260 tv.push_back((float)*(int16_t *) &buf[i]);261 } else if (t->type == GGML_TYPE_I8) {262 tv.push_back((float)*(int8_t *) &buf[i]);263 } else if (quantized) {264 tt->to_float(&buf[i], vq.data(), bs);265 tv.insert(tv.end(), vq.begin(), vq.end());266 } else {267 GGML_ABORT("fatal error");268 }269 }270 }271 }272 }273 274 return tv;275}276 277// normalized mean squared error = mse(a, b) / mse(a, 0)278static double nmse(const float * a, const float * b, size_t n) {279 double mse_a_b = 0.0;280 double mse_a_0 = 0.0;281 282 for (size_t i = 0; i < n; i++) {283 float a_i = a[i];284 float b_i = b[i];285 286 mse_a_b += (a_i - b_i) * (a_i - b_i);287 mse_a_0 += a_i * a_i;288 }289 290 return mse_a_b / mse_a_0;291}292 293// difference between 2 sets (Jaccard distance, 0 - no difference, 1 - no overlap)294template <typename T>295static double jdst(const T * a, const T * b, size_t n) {296 std::unordered_map<T, size_t> set_a;297 std::unordered_map<T, size_t> set_b;298 299 for (size_t i = 0; i < n; ++i) {300 set_a[a[i]]++;301 set_b[b[i]]++;302 }303 304 size_t diff = 0;305 306 for (const auto & p : set_a) {307 const int64_t na = p.second;308 const int64_t nb = set_b.find(p.first) != set_b.end() ? set_b.at(p.first) : 0;309 310 diff += std::abs(na - nb);311 }312 313 for (const auto & p : set_b) {314 if (set_a.find(p.first) == set_a.end()) {315 diff += p.second;316 }317 }318 319 return (double) diff / (2*n);320}321 322// maximum absolute asymmetry between a and b323// asymmetry: (a - b) / (a + b)324// This is more stable than relative error if one of the values fluctuates towards zero.325// n: number of values to compare.326// expected_vals: optional vector of expected values for a. If expected_vals is not empty, filter out all comparisons where327// a does not match any of the expected values. Needed for noncontinuous gradients where the numerical calculation can fail.328static double mean_abs_asymm(const float * a, const float * b, const size_t n, const std::vector<float> & expected_vals) {329 double sum = 0.0f;330 331 size_t nvalid = 0;332 for (size_t i = 0; i < n; i++) {333 if (!expected_vals.empty()) {334 bool matches_any = false;335 for (const float & ev : expected_vals) {336 if (fabsf(a[i] - ev) < 1e-3f) {337 matches_any = true;338 break;339 }340 }341 if (!matches_any) {342 continue;343 }344 }345 346 const float asymm = (a[i] - b[i]) / (a[i] + b[i]);347 348 sum += fabsf(asymm);349 nvalid++;350 }351 352 return sum/nvalid;353}354 355// utils for printing the variables of the test cases356 357static std::string var_to_str(const std::string & x) {358 return x;359}360 361template<typename T>362static std::string var_to_str(const T & x) {363 return std::to_string(x);364}365 366template<typename T, size_t N>367static std::string var_to_str(const T (&x)[N]) {368 std::string s = "[";369 for (size_t i = 0; i < N; i++) {370 if (i > 0) {371 s += ",";372 }373 s += var_to_str(x[i]);374 }375 s += "]";376 return s;377}378 379template<typename T, size_t N>380static std::string var_to_str(const std::array<T, N> & x) {381 std::string s = "[";382 for (size_t i = 0; i < N; i++) {383 if (i > 0) {384 s += ",";385 }386 s += var_to_str(x[i]);387 }388 s += "]";389 return s;390}391 392static std::string var_to_str(ggml_type type) {393 return ggml_type_name(type);394}395 396static std::string var_to_str(ggml_prec prec) {397 return prec == GGML_PREC_F32 ? "f32" : "def";398}399 400static std::string var_to_str(ggml_op_pool pool) {401 switch (pool) {402 case GGML_OP_POOL_AVG: return "avg";403 case GGML_OP_POOL_MAX: return "max";404 default: return std::to_string(pool);405 }406}407 408static std::string var_to_str(ggml_scale_mode mode) {409 std::string str;410 switch (mode & 0xFF) {411 case GGML_SCALE_MODE_NEAREST: str = "nearest"; break;412 case GGML_SCALE_MODE_BILINEAR: str = "bilinear"; break;413 case GGML_SCALE_MODE_BICUBIC: str = "bicubic"; break;414 default: str = std::to_string(mode); break;415 }416 if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) {417 str += "|align_corners";418 }419 if (mode & GGML_SCALE_FLAG_ANTIALIAS) {420 str += "|antialias";421 }422 return str;423}424 425#define VAR_TO_STR(x) (#x "=" + var_to_str(x))426 427#define VARS_TO_STR1(a) VAR_TO_STR(a)428#define VARS_TO_STR2(a, b) VAR_TO_STR(a) + "," + VAR_TO_STR(b)429#define VARS_TO_STR3(a, b, c) VAR_TO_STR(a) + "," + VARS_TO_STR2(b, c)430#define VARS_TO_STR4(a, b, c, d) VAR_TO_STR(a) + "," + VARS_TO_STR3(b, c, d)431#define VARS_TO_STR5(a, b, c, d, e) VAR_TO_STR(a) + "," + VARS_TO_STR4(b, c, d, e)432#define VARS_TO_STR6(a, b, c, d, e, f) VAR_TO_STR(a) + "," + VARS_TO_STR5(b, c, d, e, f)433#define VARS_TO_STR7(a, b, c, d, e, f, g) VAR_TO_STR(a) + "," + VARS_TO_STR6(b, c, d, e, f, g)434#define VARS_TO_STR8(a, b, c, d, e, f, g, h) VAR_TO_STR(a) + "," + VARS_TO_STR7(b, c, d, e, f, g, h)435#define VARS_TO_STR9(a, b, c, d, e, f, g, h, i) VAR_TO_STR(a) + "," + VARS_TO_STR8(b, c, d, e, f, g, h, i)436#define VARS_TO_STR10(a, b, c, d, e, f, g, h, i, j) VAR_TO_STR(a) + "," + VARS_TO_STR9(b, c, d, e, f, g, h, i, j)437#define VARS_TO_STR11(a, b, c, d, e, f, g, h, i, j, k) VAR_TO_STR(a) + "," + VARS_TO_STR10(b, c, d, e, f, g, h, i, j, k)438#define VARS_TO_STR12(a, b, c, d, e, f, g, h, i, j, k, l) VAR_TO_STR(a) + "," + VARS_TO_STR11(b, c, d, e, f, g, h, i, j, k, l)439#define VARS_TO_STR13(a, b, c, d, e, f, g, h, i, j, k, l, m) VAR_TO_STR(a) + "," + VARS_TO_STR12(b, c, d, e, f, g, h, i, j, k, l, m)440#define VARS_TO_STR14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) VAR_TO_STR(a) + "," + VARS_TO_STR13(b, c, d, e, f, g, h, i, j, k, l, m, n)441#define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o)442#define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)443 444#ifdef GGML_USE_SYCL445static bool inline _isinf(float f) {446 return (*(uint32_t *)&f & 0x7fffffff) == 0x7f800000;447}448#else449static bool inline _isinf(float f) { return std::isinf(f); }450#endif451 452// accept FLT_MAX as infinity453static bool isinf_or_max(float f) {454 return _isinf(f) || f == FLT_MAX || f == -FLT_MAX;455}456 457static bool ggml_is_view_op(enum ggml_op op) {458 return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;459}460 461static bool backend_has_feature(ggml_backend_t backend, const char * feature_name) {462 ggml_backend_dev_t dev = ggml_backend_get_device(backend);463 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);464 465 auto get_features = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");466 if (!get_features) {467 return false;468 }469 470 const ggml_backend_feature * features = get_features(reg);471 if (!features) {472 return false;473 }474 475 for (const ggml_backend_feature * f = features; f->name; ++f) {476 if (strcmp(f->name, feature_name) == 0 && strcmp(f->value, "1") == 0) {477 return true;478 }479 }480 return false;481}482 483enum test_mode {484 MODE_TEST,485 MODE_PERF,486 MODE_GRAD,487 MODE_SUPPORT,488};489 490// Output format support similar to llama-bench491enum output_formats { CONSOLE, SQL, CSV };492 493static const char * output_format_str(output_formats format) {494 switch (format) {495 case CONSOLE:496 return "console";497 case SQL:498 return "sql";499 case CSV:500 return "csv";501 default:502 GGML_ABORT("invalid output format");503 }504}505 506static bool output_format_from_str(const std::string & s, output_formats & format) {507 if (s == "console") {508 format = CONSOLE;509 } else if (s == "sql") {510 format = SQL;511 } else if (s == "csv") {512 format = CSV;513 } else {514 return false;515 }516 return true;517}518 519// Test result structure for SQL output520struct test_result {521 std::string test_time;522 std::string build_commit;523 std::string backend_name;524 std::string op_name;525 std::string op_params;526 std::string test_mode;527 bool supported;528 bool passed;529 std::string error_message;530 double time_us;531 double flops;532 double bandwidth_gb_s;533 size_t memory_kb;534 int n_runs;535 std::string device_description;536 std::string backend_reg_name;537 538 test_result() {539 // Initialize with default values540 time_us = 0.0;541 flops = 0.0;542 bandwidth_gb_s = 0.0;543 memory_kb = 0;544 n_runs = 0;545 supported = false;546 passed = false;547 548 // Set test time549 time_t t = time(NULL);550 char buf[32];551 std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));552 test_time = buf;553 554 // Set build info555 build_commit = ggml_commit();556 }557 558 test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params,559 const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "",560 double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0,561 int n_runs = 0, const std::string & device_description = "", const std::string & backend_reg_name = "") :562 backend_name(backend_name),563 op_name(op_name),564 op_params(op_params),565 test_mode(test_mode),566 supported(supported),567 passed(passed),568 error_message(error_message),569 time_us(time_us),570 flops(flops),571 bandwidth_gb_s(bandwidth_gb_s),572 memory_kb(memory_kb),573 n_runs(n_runs),574 device_description(device_description),575 backend_reg_name(backend_reg_name) {576 // Set test time577 time_t t = time(NULL);578 char buf[32];579 std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t));580 test_time = buf;581 582 // Set build info583 build_commit = ggml_commit();584 }585 586 static const std::vector<std::string> & get_fields() {587 static const std::vector<std::string> fields = {588 "test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported",589 "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs",590 "device_description", "backend_reg_name"591 };592 return fields;593 }594 595 enum field_type { STRING, BOOL, INT, FLOAT };596 597 static field_type get_field_type(const std::string & field) {598 if (field == "supported" || field == "passed") {599 return BOOL;600 }601 if (field == "memory_kb" || field == "n_runs") {602 return INT;603 }604 if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") {605 return FLOAT;606 }607 return STRING;608 }609 610 std::vector<std::string> get_values() const {611 return { test_time,612 build_commit,613 backend_name,614 op_name,615 op_params,616 test_mode,617 std::to_string(supported),618 std::to_string(passed),619 error_message,620 std::to_string(time_us),621 std::to_string(flops),622 std::to_string(bandwidth_gb_s),623 std::to_string(memory_kb),624 std::to_string(n_runs),625 device_description,626 backend_reg_name };627 }628};629 630// Printer classes for different output formats631enum class test_status_t { NOT_SUPPORTED, OK, FAIL, SKIPPED };632 633struct test_operation_info {634 std::string op_name;635 std::string op_params;636 std::string backend_name;637 test_status_t status = test_status_t::OK;638 std::string failure_reason;639 640 // Additional information fields that were previously in separate structs641 std::string error_component;642 std::string error_details;643 644 // Gradient info645 int64_t gradient_index = -1;646 std::string gradient_param_name;647 float gradient_value = 0.0f;648 649 // MAA error info650 double maa_error = 0.0;651 double maa_threshold = 0.0;652 653 // Flags for different types of information654 bool has_error = false;655 bool has_gradient_info = false;656 bool has_maa_error = false;657 bool is_compare_failure = false;658 bool is_large_tensor_skip = false;659 660 test_operation_info() = default;661 662 test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name,663 test_status_t status = test_status_t::OK, const std::string & failure_reason = "") :664 op_name(op_name),665 op_params(op_params),666 backend_name(backend_name),667 status(status),668 failure_reason(failure_reason) {}669 670 // Set error information671 void set_error(const std::string & component, const std::string & details) {672 has_error = true;673 error_component = component;674 error_details = details;675 if (status == test_status_t::OK) {676 status = test_status_t::FAIL;677 }678 }679 680 // Set gradient information681 void set_gradient_info(int64_t index, const std::string & param_name, float value) {682 has_gradient_info = true;683 gradient_index = index;684 gradient_param_name = param_name;685 gradient_value = value;686 if (status == test_status_t::OK) {687 status = test_status_t::FAIL;688 }689 }690 691 // Set MAA error information692 void set_maa_error(double error, double threshold) {693 has_maa_error = true;694 maa_error = error;695 maa_threshold = threshold;696 if (status == test_status_t::OK) {697 status = test_status_t::FAIL;698 }699 }700 701 // Set compare failure702 void set_compare_failure() {703 is_compare_failure = true;704 if (status == test_status_t::OK) {705 status = test_status_t::FAIL;706 }707 }708 709 // Set large tensor skip710 void set_large_tensor_skip() { is_large_tensor_skip = true; }711};712 713struct test_summary_info {714 size_t tests_passed;715 size_t tests_total;716 bool is_backend_summary = false; // true for backend summary, false for test summary717 718 test_summary_info() = default;719 720 test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) :721 tests_passed(tests_passed),722 tests_total(tests_total),723 is_backend_summary(is_backend_summary) {}724};725 726struct testing_start_info {727 size_t device_count;728 729 testing_start_info() = default;730 731 testing_start_info(size_t device_count) : device_count(device_count) {}732};733 734struct backend_init_info {735 size_t device_index;736 size_t total_devices;737 std::string device_name;738 bool skipped = false;739 std::string skip_reason;740 std::string description;741 size_t memory_total_mb = 0;742 size_t memory_free_mb = 0;743 bool has_memory_info = false;744 745 backend_init_info() = default;746 747 backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false,748 const std::string & skip_reason = "", const std::string & description = "",749 size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) :750 device_index(device_index),751 total_devices(total_devices),752 device_name(device_name),753 skipped(skipped),754 skip_reason(skip_reason),755 description(description),756 memory_total_mb(memory_total_mb),757 memory_free_mb(memory_free_mb),758 has_memory_info(has_memory_info) {}759};760 761struct backend_status_info {762 std::string backend_name;763 test_status_t status;764 765 backend_status_info() = default;766 767 backend_status_info(const std::string & backend_name, test_status_t status) :768 backend_name(backend_name),769 status(status) {}770};771 772struct overall_summary_info {773 size_t backends_passed;774 size_t backends_total;775 bool all_passed;776 777 overall_summary_info() = default;778 779 overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) :780 backends_passed(backends_passed),781 backends_total(backends_total),782 all_passed(all_passed) {}783};784 785struct printer {786 virtual ~printer() {}787 788 FILE * fout = stdout;789 790 virtual void print_header() {}791 792 virtual void print_test_result(const test_result & result) = 0;793 794 virtual void print_footer() {}795 796 virtual void print_operation(const test_operation_info & info) { (void) info; }797 798 virtual void print_summary(const test_summary_info & info) { (void) info; }799 800 virtual void print_testing_start(const testing_start_info & info) { (void) info; }801 802 virtual void print_backend_init(const backend_init_info & info) { (void) info; }803 804 virtual void print_backend_status(const backend_status_info & info) { (void) info; }805 806 virtual void print_overall_summary(const overall_summary_info & info) { (void) info; }807 808 virtual void print_failed_tests(const std::vector<std::string> & failed_tests) { (void) failed_tests; }809};810 811struct console_printer : public printer {812 void print_test_result(const test_result & result) override {813 if (result.test_mode == "test") {814 print_test_console(result);815 } else if (result.test_mode == "perf") {816 print_perf_console(result);817 } else if (result.test_mode == "support") {818 print_support_console(result);819 }820 }821 822 void print_operation(const test_operation_info & info) override {823 printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str());824 fflush(stdout);825 826 // Handle large tensor skip first827 if (info.is_large_tensor_skip) {828 printf("skipping large tensors for speed \n");829 return;830 }831 832 // Handle not supported status833 if (info.status == test_status_t::NOT_SUPPORTED) {834 if (!info.failure_reason.empty()) {835 printf("not supported [%s]\n", info.failure_reason.c_str());836 } else {837 printf("not supported [%s]\n", info.backend_name.c_str());838 }839 return;840 }841 842 // Handle errors and additional information843 if (info.has_error) {844 if (info.error_component == "allocation") {845 fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str());846 } else if (info.error_component == "backend") {847 fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str());848 } else {849 fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str());850 }851 }852 853 // Handle gradient info854 if (info.has_gradient_info) {855 printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index,856 info.gradient_param_name.c_str(), info.gradient_value);857 }858 859 // Handle MAA error860 if (info.has_maa_error) {861 printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold);862 }863 864 // Handle compare failure865 if (info.is_compare_failure) {866 printf("compare failed ");867 }868 869 // Print final status870 if (info.status == test_status_t::OK) {871 printf("\033[1;32mOK\033[0m\n");872 } else {873 printf("\033[1;31mFAIL\033[0m\n");874 }875 }876 877 void print_summary(const test_summary_info & info) override {878 if (info.is_backend_summary) {879 printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total);880 } else {881 printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total);882 }883 }884 885 void print_backend_status(const backend_status_info & info) override {886 printf(" Backend %s: ", info.backend_name.c_str());887 if (info.status == test_status_t::OK) {888 printf("\033[1;32mOK\033[0m\n");889 } else {890 printf("\033[1;31mFAIL\033[0m\n");891 }892 }893 894 void print_testing_start(const testing_start_info & info) override {895 printf("Testing %zu devices\n\n", info.device_count);896 }897 898 void print_backend_init(const backend_init_info & info) override {899 printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str());900 901 if (info.skipped) {902 printf(" %s\n", info.skip_reason.c_str());903 return;904 }905 906 if (!info.description.empty()) {907 printf(" Device description: %s\n", info.description.c_str());908 }909 910 if (info.has_memory_info) {911 printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb);912 }913 914 printf("\n");915 }916 917 void print_overall_summary(const overall_summary_info & info) override {918 printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total);919 if (info.all_passed) {920 printf("\033[1;32mOK\033[0m\n");921 } else {922 printf("\033[1;31mFAIL\033[0m\n");923 }924 }925 926 void print_failed_tests(const std::vector<std::string> & failed_tests) override {927 if (failed_tests.empty()) {928 return;929 }930 931 printf("\nFailing tests:\n");932 for (const auto & test_name : failed_tests) {933 printf(" %s\n", test_name.c_str());934 }935 }936 937 private:938 void print_test_console(const test_result & result) {939 printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());940 fflush(stdout);941 942 if (!result.supported) {943 printf("not supported [%s] ", result.backend_name.c_str());944 printf("\n");945 return;946 }947 948 if (result.passed) {949 printf("\033[1;32mOK\033[0m\n");950 } else {951 printf("\033[1;31mFAIL\033[0m\n");952 }953 }954 955 void print_perf_console(const test_result & result) {956 int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());957 fflush(stdout);958 959 if (!result.supported) {960 printf("not supported\n");961 return;962 }963 964 // align while also leaving some margin for variations in parameters965 int align = 8;966 int last = (len + align - 1) / align * align;967 if (last - len < 5) {968 last += align;969 }970 printf("%*s", last - len, "");971 972 printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us);973 974 if (result.flops > 0) {975 auto format_flops = [](double flops) -> std::string {976 char buf[256];977 if (flops >= 1e12) {978 snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12);979 } else if (flops >= 1e9) {980 snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9);981 } else if (flops >= 1e6) {982 snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6);983 } else {984 snprintf(buf, sizeof(buf), "%6.2f kFLOP", flops / 1e3);985 }986 return buf;987 };988 uint64_t op_flops_per_run = result.flops * result.time_us / 1e6;989 printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(),990 format_flops(result.flops).c_str());991 } else {992 printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s);993 }994 printf("\n");995 }996 997 void print_support_console(const test_result & result) {998 printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());999 fflush(stdout);1000 1001 if (result.supported) {1002 printf("\033[1;32mSUPPORTED\033[0m\n");1003 } else {1004 printf("\033[1;31mNOT SUPPORTED\033[0m\n");1005 }1006 }1007};1008 1009struct sql_printer : public printer {1010 static std::string get_sql_field_type(const std::string & field) {1011 switch (test_result::get_field_type(field)) {1012 case test_result::STRING:1013 return "TEXT";1014 case test_result::BOOL:1015 case test_result::INT:1016 return "INTEGER";1017 case test_result::FLOAT:1018 return "REAL";1019 default:1020 GGML_ABORT("invalid field type");1021 }1022 }1023 1024 void print_header() override {1025 std::vector<std::string> fields = test_result::get_fields();1026 fprintf(fout, "CREATE TABLE IF NOT EXISTS test_backend_ops (\n");1027 for (size_t i = 0; i < fields.size(); i++) {1028 fprintf(fout, " %s %s%s\n", fields[i].c_str(), get_sql_field_type(fields[i]).c_str(),1029 i < fields.size() - 1 ? "," : "");1030 }1031 fprintf(fout, ");\n\n");1032 }1033 1034 void print_test_result(const test_result & result) override {1035 fprintf(fout, "INSERT INTO test_backend_ops (");1036 std::vector<std::string> fields = test_result::get_fields();1037 for (size_t i = 0; i < fields.size(); i++) {1038 fprintf(fout, "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : "");1039 }1040 fprintf(fout, ") VALUES (");1041 std::vector<std::string> values = result.get_values();1042 for (size_t i = 0; i < values.size(); i++) {1043 fprintf(fout, "'%s'%s", values[i].c_str(), i < values.size() - 1 ? ", " : "");1044 }1045 fprintf(fout, ");\n");1046 }1047};1048 1049struct csv_printer : public printer {1050 void print_header() override {1051 1052 std::vector<std::string> fields = test_result::get_fields();1053 std::vector<std::string> fields_csv = get_fields_csv();1054 for (size_t i = 0; i < fields.size(); i++) {1055 if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) {1056 continue;1057 }1058 printf("\"%s\"%s", fields[i].c_str(), i < fields.size() - 1 ? "," : "");1059 }1060 printf("\n");1061 }1062 1063 void print_test_result(const test_result & result) override {1064 1065 std::vector<std::string> values = result.get_values();1066 std::vector<std::string> fields = test_result::get_fields();1067 std::vector<std::string> fields_csv = get_fields_csv();1068 1069 for (size_t i = 0; i < values.size(); i++) {1070 1071 if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) {1072 continue;1073 }1074 1075 // Escape quotes and wrap in quotes for CSV1076 std::string escaped_value = values[i];1077 size_t pos = 0;1078 while ((pos = escaped_value.find("\"", pos)) != std::string::npos) {1079 escaped_value.replace(pos, 1, "\"\"");1080 pos += 2;1081 }1082 printf("\"%s\"%s", escaped_value.c_str(), i < values.size() - 1 ? "," : "");1083 }1084 printf("\n");1085 }1086 1087 static std::vector<std::string> get_fields_csv() {1088 return {1089 "op_name",1090 "op_params",1091 "supported",1092 "error_message",1093 "test_mode",1094 "backend_reg_name",1095 "backend_name",1096 };1097 }1098 1099};1100 1101static std::unique_ptr<printer> create_printer(output_formats format) {1102 switch (format) {1103 case CONSOLE:1104 return std::make_unique<console_printer>();1105 case SQL:1106 return std::make_unique<sql_printer>();1107 case CSV:1108 return std::make_unique<csv_printer>();1109 }1110 GGML_ABORT("invalid output format");1111}1112 1113struct test_case {1114 virtual ~test_case() {}1115 1116 virtual std::string op_desc(ggml_tensor * t) {1117 return ggml_op_desc(t);1118 }1119 1120 virtual std::string vars() {1121 return "";1122 }1123 1124 virtual ggml_tensor * build_graph(ggml_context * ctx) = 0;1125 1126 virtual double max_nmse_err() {1127 return 1e-7;1128 }1129 1130 virtual double max_nmse_err(ggml_backend_t backend) {1131 GGML_UNUSED(backend);1132 return max_nmse_err();1133 }1134 1135 virtual double max_maa_err() {1136 return 1e-4;1137 }1138 1139 virtual double max_err() {1140 return max_nmse_err();1141 }1142 1143 virtual double max_err(ggml_backend_t backend) {1144 return max_nmse_err(backend);1145 }1146 1147 virtual double err(const float * a, const float * b, size_t n) {1148 return nmse(a, b, n);1149 }1150 1151 virtual float grad_eps() {1152 return 1e-1f;1153 }1154 1155 // If false, estimate gradient with 2 points, neglects 3rd order derivative and higher.1156 // If true, estimate gradient with 4 points, neglects 5th order derivative and higher.1157 virtual bool grad_precise() {1158 return false;1159 }1160 1161 // Skip gradient checks if total number of gradients to be checked is larger than this (to speed up the tests).1162 virtual int64_t grad_nmax() {1163 return 10000;1164 }1165 1166 // No effect if empty.1167 // If not empty, skip all gradient checks where the numerical result does not match any of the values.1168 // Needed for dealing with noncontinuous gradients (e.g. ReLU) where estimation using finite differences is unreliable.1169 virtual std::vector<float> grad_expect() {1170 return {};1171 }1172 1173 virtual void initialize_tensors(ggml_context * ctx) {1174 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {1175 init_tensor_uniform(t);1176 }1177 }1178 1179 virtual size_t op_size(ggml_tensor * t) {1180 size_t size = ggml_nbytes(t);1181 // add source tensors1182 for (int i = 0; i < GGML_MAX_SRC; i++) {1183 if (t->src[i] != NULL) {1184 size += ggml_nbytes(t->src[i]);1185 }1186 }1187 return size;1188 }1189 1190 virtual uint64_t op_flops(ggml_tensor * t) {1191 GGML_UNUSED(t);1192 return 0;1193 }1194 1195 virtual bool run_whole_graph() { return false; }1196 virtual std::vector<ggml_tensor *> fusion_test_nodes() { return {}; }1197 1198 ggml_cgraph * gf = nullptr;1199 ggml_cgraph * gb = nullptr;1200