KBaba7/llama.cpp

1#include "ggml.h"2#include "ggml-cpu.h"3#include "ggml-backend.h"4 5#include <chrono>6#include <iostream>7#include <cstdio>8#include <cstdlib>9#include <cassert>10#include <vector>11 12#define MAX_NARGS 213 14int main(int argc, char *argv[]) {15 16    int n_threads = 4;17    int n_rounds  = 100;18 19    if (argc > 1) {20        n_threads = std::atoi(argv[1]);21    }22 23    if (argc > 2) {24        n_rounds  = std::atoi(argv[2]);25    }26 27    struct ggml_init_params params = {28        /* .mem_size   = */ 1024*1024*1024,29        /* .mem_buffer = */ NULL,30        /* .no_alloc   = */ false,31    };32 33    struct ggml_context * ctx = ggml_init(params);34 35    // Create graph36    struct ggml_cgraph * gf = ggml_new_graph(ctx);37 38    // Lots of small, parallel ops where barriers in between will dominate39    struct ggml_tensor * out = ggml_new_tensor_1d(ctx, GGML_TYPE_F32,  64);40    for (int i = 0; i < 1000; i++) {41        struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 64, 128);42        out = ggml_mul_mat(ctx, a, out);43 44        struct ggml_tensor * d = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 128, 64);45        out = ggml_mul_mat(ctx, d, out);46    }47 48    ggml_build_forward_expand(gf, out);49    int n_nodes = ggml_graph_n_nodes(gf);50 51    // Create threadpool52    struct ggml_threadpool_params tpp  = ggml_threadpool_params_default(n_threads);53    struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);54    if (!threadpool) {55        fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);56        exit(1);57    }58 59    // Create compute plan60    struct ggml_cplan cplan = ggml_graph_plan(gf, n_threads, threadpool);61 62    std::vector<uint8_t> work_data(cplan.work_size);63    cplan.work_data = work_data.data();64 65    std::cerr << "graph-compute with"66              << "\n n_threads: " << n_threads67              << "\n   n_nodes: " << n_nodes68              << "\n  n_rounds: " << n_rounds69              << "\n";70    // ggml_graph_print(gf);71 72    // Warmup73    ggml_graph_compute(gf, &cplan);74 75    auto t0 = std::chrono::high_resolution_clock::now();76 77    for (int i=0; i < n_rounds; i++) {78        ggml_graph_compute(gf, &cplan);79    }80 81    auto t1 = std::chrono::high_resolution_clock::now();82 83    auto usec = std::chrono::duration_cast<std::chrono::microseconds>(t1-t0).count();84    auto nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(t1-t0).count();85    std::cerr << "graph-compute took " << usec << " usec "86              << "\n " << (float) usec / n_rounds << " usec per-iter"87              << "\n " << (float) nsec / (n_rounds * n_nodes) << " nsec per-node"88              << "\n";89 90    ggml_threadpool_free(threadpool);91    ggml_free(ctx);92 93    return 0;94}95