Brunobkr/llama.cpp_AlgMor24_github

1#include "ggml-opt.h"2 3#include "ggml.h"4#include "ggml-alloc.h"5#include "ggml-backend.h"6#include "ggml-impl.h"7 8#include <algorithm>9#include <cmath>10#include <cstdint>11#include <cinttypes>12#include <map>13#include <random>14#include <vector>15 16struct ggml_opt_dataset {17    struct ggml_context   * ctx    = nullptr;18    ggml_backend_buffer_t   buf    = nullptr;19    struct ggml_tensor    * data   = nullptr;20    struct ggml_tensor    * labels = nullptr;21 22    int64_t ndata       = -1;23    int64_t ndata_shard = -1;24    size_t  nbs_data    = -1;25    size_t  nbs_labels  = -1;26 27    std::vector<int64_t> permutation;28};29 30struct ggml_opt_context {31    ggml_backend_sched_t       backend_sched        = nullptr;32    ggml_cgraph              * allocated_graph      = nullptr;33    ggml_cgraph              * allocated_graph_copy = nullptr;34    struct ggml_context      * ctx_static           = nullptr;35    struct ggml_context      * ctx_cpu              = nullptr;36    struct ggml_context      * ctx_compute          = nullptr;37    struct ggml_context      * ctx_copy             = nullptr;38    ggml_backend_buffer_t      buf_static           = nullptr;39    ggml_backend_buffer_t      buf_cpu              = nullptr;40    std::mt19937               rng;41    enum ggml_opt_loss_type    loss_type;42    enum ggml_opt_build_type   build_type;43    enum ggml_opt_build_type   build_type_alloc;44 45    struct ggml_tensor * inputs  = nullptr;46    struct ggml_tensor * outputs = nullptr;47    struct ggml_tensor * labels  = nullptr;48 49    struct ggml_tensor * loss     = nullptr;50    struct ggml_tensor * pred     = nullptr;51    struct ggml_tensor * ncorrect = nullptr;52 53    struct ggml_cgraph * gf      = nullptr;54    struct ggml_cgraph * gb_grad = nullptr;55    struct ggml_cgraph * gb_opt  = nullptr;56    bool static_graphs           = false;57    bool eval_ready              = false;58    std::vector<struct ggml_tensor *> grad_accs;59    std::vector<struct ggml_tensor *> grad_m;60    std::vector<struct ggml_tensor *> grad_v;61 62    int64_t iter               = 1;63    int32_t opt_period         = 1;64    int32_t opt_i              = 0;65    bool    loss_per_datapoint = false;66 67    ggml_opt_get_optimizer_params get_opt_pars    = nullptr;68    void *                        get_opt_pars_ud = nullptr;69    struct ggml_tensor *          opt_step_params = nullptr; // Stores output of get_opt_pars.70 71    enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;72};73 74struct ggml_opt_result {75    int64_t              ndata    = 0;76    std::vector<float>   loss;77    std::vector<int32_t> pred;78    int64_t              ncorrect = 0;79 80    int64_t opt_period         = -1;81    bool    loss_per_datapoint = false;82};83 84// ====== Dataset ======85 86ggml_opt_dataset_t ggml_opt_dataset_init(87        enum ggml_type type_data,88        enum ggml_type type_label,89        int64_t        ne_datapoint,90        int64_t        ne_label,91        int64_t        ndata,92        int64_t        ndata_shard) {93    GGML_ASSERT(ne_datapoint >  0);94    GGML_ASSERT(ne_label     >= 0);95    GGML_ASSERT(ndata        >  0);96    GGML_ASSERT(ndata_shard  >  0);97 98    ggml_opt_dataset_t result = new ggml_opt_dataset;99    result->ndata       = ndata;100    result->ndata_shard = ndata_shard;101 102    {103        struct ggml_init_params params = {104            /*.mem_size   =*/ 2*ggml_tensor_overhead(),105            /*.mem_buffer =*/ nullptr,106            /*.no_alloc   =*/ true,107        };108        result->ctx = ggml_init(params);109    }110 111    result->data = ggml_new_tensor_2d(result->ctx, type_data, ne_datapoint, ndata);112    result->nbs_data = ggml_nbytes(result->data) * ndata_shard/ndata;113 114    if (ne_label > 0) {115        result->labels = ggml_new_tensor_2d(result->ctx, type_label, ne_label, ndata);116        result->nbs_labels = ggml_nbytes(result->labels) * ndata_shard/ndata;117    } else {118        result->labels = nullptr;119        result->nbs_labels = 0;120    }121 122    result->buf = ggml_backend_alloc_ctx_tensors_from_buft(result->ctx, ggml_backend_cpu_buffer_type());123 124    const int64_t nshards = ndata/ndata_shard;125    result->permutation.resize(nshards);126    for (int64_t i = 0; i < nshards; ++i) {127        result->permutation[i] = i;128    }129    return result;130}131 132void ggml_opt_dataset_free(ggml_opt_dataset_t dataset) {133    ggml_backend_buffer_free(dataset->buf);134    ggml_free(dataset->ctx);135    delete dataset;136}137 138int64_t ggml_opt_dataset_ndata(ggml_opt_dataset_t dataset) {139    return dataset->ndata;140}141 142struct ggml_tensor * ggml_opt_dataset_data(ggml_opt_dataset_t dataset) {143    return dataset->data;144}145 146struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset) {147    return dataset->labels;148}149 150void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata) {151    GGML_ASSERT(idata <= dataset->ndata);152 153    if (idata < 0) {154        std::shuffle(dataset->permutation.begin(), dataset->permutation.end(), opt_ctx->rng);155        return;156    }157 158    GGML_ASSERT(idata % dataset->ndata_shard == 0);159    const int64_t ishard_max = idata / dataset->ndata_shard;160    std::shuffle(dataset->permutation.begin(), dataset->permutation.begin() + ishard_max, opt_ctx->rng);161}162 163void ggml_opt_dataset_get_batch(ggml_opt_dataset_t dataset, struct ggml_tensor * data_batch, struct ggml_tensor * labels_batch, int64_t ibatch) {164    GGML_ASSERT(   data_batch && ggml_is_contiguous(data_batch));165    GGML_ASSERT(!labels_batch || ggml_is_contiguous(labels_batch));166    GGML_ASSERT((labels_batch == nullptr) == (dataset->labels == nullptr));167    GGML_ASSERT(                   data_batch->type == dataset->data->type);168    GGML_ASSERT(!labels_batch || labels_batch->type == dataset->labels->type);169 170    const size_t nb_data_batch = ggml_nbytes(data_batch);171    GGML_ASSERT(nb_data_batch % dataset->nbs_data == 0);172    const int64_t shards_per_batch = nb_data_batch / dataset->nbs_data;173 174    if (labels_batch) {175        const size_t nb_labels_batch = ggml_nbytes(labels_batch);176        GGML_ASSERT(nb_labels_batch == shards_per_batch*dataset->nbs_labels);177    }178 179    GGML_ASSERT((ibatch + 1)*shards_per_batch <= int64_t(dataset->permutation.size()));180 181    for (int64_t ishard_batch = 0; ishard_batch < shards_per_batch; ++ishard_batch) {182        const int64_t ishard = dataset->permutation[ibatch*shards_per_batch + ishard_batch];183 184        const char * ptr_data = (const char *) dataset->data->data + ishard*dataset->nbs_data;185        ggml_backend_tensor_set(data_batch, ptr_data, ishard_batch*dataset->nbs_data, dataset->nbs_data);186 187        if (!labels_batch) {188            continue;189        }190 191        const char * ptr_labels = (const char *) dataset->labels->data + ishard*dataset->nbs_labels;192        ggml_backend_tensor_set(labels_batch, ptr_labels, ishard_batch*dataset->nbs_labels, dataset->nbs_labels);193    }194}195 196void ggml_opt_dataset_get_batch_host(ggml_opt_dataset_t dataset, void * data_batch, size_t nb_data_batch, void * labels_batch, int64_t ibatch) {197    GGML_ASSERT((labels_batch == nullptr) == (dataset->labels == nullptr));198    GGML_ASSERT(nb_data_batch % dataset->nbs_data == 0);199 200    const int64_t shards_per_batch = nb_data_batch / dataset->nbs_data;201 202    GGML_ASSERT((ibatch + 1)*shards_per_batch <= int64_t(dataset->permutation.size()));203 204    for (int64_t ishard_batch = 0; ishard_batch < shards_per_batch; ++ishard_batch) {205        const int64_t ishard = dataset->permutation[ibatch*shards_per_batch + ishard_batch];206 207        const char * ptr_data       = (const char *) dataset->data->data + ishard      *dataset->nbs_data;208        char       * ptr_data_batch = (char       *) data_batch          + ishard_batch*dataset->nbs_data;209        memcpy(ptr_data_batch, ptr_data, dataset->nbs_data);210 211        if (!labels_batch) {212            continue;213        }214 215        const char * ptr_labels       = (const char *) dataset->labels->data + ishard      *dataset->nbs_labels;216        char       * ptr_labels_batch = (char       *) labels_batch          + ishard_batch*dataset->nbs_labels;217        memcpy(ptr_labels_batch, ptr_labels, dataset->nbs_labels);218    }219}220 221// ====== Model / Context ======222 223struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata) {224    GGML_UNUSED(userdata);225 226    ggml_opt_optimizer_params result;227 228    result.adamw.alpha = 0.001f;229    result.adamw.beta1 = 0.9f;230    result.adamw.beta2 = 0.999f;231    result.adamw.eps   = 1e-8f;232    result.adamw.wd    = 0.0f;233 234    result.sgd.alpha   = 1e-3f;235    result.sgd.wd      = 0.0f;236 237    return result;238}239 240 241struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata) {242    return *((struct ggml_opt_optimizer_params *) userdata);243}244 245struct ggml_opt_params ggml_opt_default_params(246        ggml_backend_sched_t      backend_sched,247        enum ggml_opt_loss_type   loss_type) {248    return {249        /*backend_sched   =*/ backend_sched,250        /*ctx_compute     =*/ nullptr,251        /*inputs          =*/ nullptr,252        /*logits          =*/ nullptr,253        /*loss_type       =*/ loss_type,254        /*build_type      =*/ GGML_OPT_BUILD_TYPE_OPT,255        /*opt_period      =*/ 1,256        /*get_opt_pars    =*/ ggml_opt_get_default_optimizer_params,257        /*get_opt_pars_ud =*/ nullptr,258        /*optimizer       =*/ GGML_OPT_OPTIMIZER_TYPE_ADAMW,259    };260}261 262static ggml_tensor * map_tensor(std::map<ggml_tensor *, ggml_tensor *> & tensor_map, ggml_context * ctx, ggml_tensor * tensor) {263    if (!tensor) {264        return nullptr;265    }266 267    if (tensor_map.find(tensor) != tensor_map.end()) {268        return tensor_map[tensor];269    }270 271    ggml_tensor * new_tensor = ggml_dup_tensor(ctx, tensor);272    tensor_map[tensor] = new_tensor;273 274    new_tensor->op = tensor->op;275    for (int i = 0; i < GGML_MAX_DIMS; i++) {276        new_tensor->nb[i] = tensor->nb[i];277    }278    new_tensor->flags = tensor->flags;279    memcpy(new_tensor->op_params, tensor->op_params, sizeof(tensor->op_params));280    strcpy(new_tensor->name, tensor->name);281    new_tensor->data = tensor->data;282    new_tensor->buffer = tensor->buffer;283    new_tensor->extra = tensor->extra;284    new_tensor->view_offs = tensor->view_offs;285    new_tensor->view_src = map_tensor(tensor_map, ctx, tensor->view_src);286    for (int i = 0; i < GGML_MAX_SRC; i++) {287        new_tensor->src[i] = map_tensor(tensor_map, ctx, tensor->src[i]);288    }289 290    return new_tensor;291}292 293static ggml_cgraph * dup_graph(ggml_context * ctx, ggml_cgraph * src) {294    std::map<ggml_tensor *, ggml_tensor *> tensor_map;295 296    ggml_cgraph * dst = ggml_new_graph_custom(ctx, src->size, /*grads =*/ true);297 298    for (int i = 0; i < src->n_leafs; i++) {299        ggml_build_forward_expand(dst, map_tensor(tensor_map, ctx, src->leafs[i]));300    }301    GGML_ASSERT(dst->n_leafs == src->n_leafs);302    for (int i = 0; i < src->n_nodes; i++) {303        ggml_build_forward_expand(dst, map_tensor(tensor_map, ctx, src->nodes[i]));304    }305    GGML_ASSERT(dst->n_nodes == src->n_nodes);306    for (int i = 0; i < src->n_nodes; ++i) {307        const size_t igrad_src = ggml_hash_find(&src->visited_hash_set, src->nodes[i]);308        const size_t igrad_dst = ggml_hash_find(&dst->visited_hash_set, dst->nodes[i]);309 310        GGML_ASSERT(igrad_src != GGML_HASHSET_FULL);311        GGML_ASSERT(ggml_bitset_get(src->visited_hash_set.used, igrad_src));312        GGML_ASSERT(igrad_dst != GGML_HASHSET_FULL);313        GGML_ASSERT(ggml_bitset_get(dst->visited_hash_set.used, igrad_dst));314 315        dst->grads[igrad_dst]     = src->grads[igrad_src];316        dst->grad_accs[igrad_dst] = src->grad_accs[igrad_src];317    }318 319    return dst;320}321 322static void ggml_opt_build(ggml_opt_context_t opt_ctx) {323    GGML_ASSERT(opt_ctx->ctx_compute && "no compute context set, either use static graphs or set one with ggml_opt_prepare_alloc");324    GGML_ASSERT((!opt_ctx->static_graphs || opt_ctx->inputs->data) && "when using static graphs the inputs must be allocated statically");325 326    const enum ggml_opt_optimizer_type optimizer = opt_ctx->optimizer;327 328    const bool accumulate = opt_ctx->build_type_alloc >= GGML_OPT_BUILD_TYPE_GRAD &&329        !(opt_ctx->static_graphs && opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_OPT && opt_ctx->opt_period == 1);330 331    const bool need_momenta = opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_OPT &&332        opt_ctx->optimizer == GGML_OPT_OPTIMIZER_TYPE_ADAMW;333 334    ggml_set_input(opt_ctx->inputs);335    ggml_set_output(opt_ctx->outputs);336 337    int n_param = 0;338    for (int i = 0; i < opt_ctx->gf->n_nodes; ++i) {339        const struct ggml_tensor * node = opt_ctx->gf->nodes[i];340        if (node->flags & GGML_TENSOR_FLAG_PARAM) {341            n_param++;342        }343        GGML_ASSERT(!(node->flags & GGML_TENSOR_FLAG_LOSS) && "support for extra loss terms not implemented");344    }345 346    if (!opt_ctx->ctx_static) {347        // The static context is used for:348        //   - gradients (1 per loss, 1 tensor per param if using gradient accumulation)349        //   - optimizer momenta (2 tensors per param)350        //   - labels (if using static graphs)351        //   - loss (if using static graphs, up to 5 tensors)352        //   - pred (if using static graphs)353        //   - ncorrect (if using static graphs, 2 tensors).354        constexpr size_t n_loss = 1;355        const size_t tensors_per_param = (accumulate ? 1 : 0) + (need_momenta ? 2 : 0);356        const size_t tensors_const = opt_ctx->static_graphs ? 9 : 0;357        const size_t size_meta = (n_loss + tensors_per_param*n_param + tensors_const) * ggml_tensor_overhead();358        struct ggml_init_params params = {359            /*.mem_size   =*/ size_meta,360            /*.mem_buffer =*/ nullptr,361            /*.no_alloc   =*/ true,362        };363        opt_ctx->ctx_static = ggml_init(params);364    }365    GGML_ASSERT(opt_ctx->build_type <= opt_ctx->build_type_alloc);366 367    {368        // The cpu context is allocated statically if using static graphs, dynamically otherwise.369        // It is used for:370        //   - optimizer parameters (1 shared for all optimizer invocations)371        const size_t size_meta = 1 * ggml_tensor_overhead();372        struct ggml_init_params params = {373            /*.mem_size   =*/ size_meta,374            /*.mem_buffer =*/ nullptr,375            /*.no_alloc   =*/ true,376        };377        ggml_free(opt_ctx->ctx_cpu);378        opt_ctx->ctx_cpu = ggml_init(params);379 380        ggml_backend_buffer_free(opt_ctx->buf_cpu);381        opt_ctx->buf_cpu = nullptr;382    }383 384    struct ggml_context * ctx_results = opt_ctx->static_graphs ? opt_ctx->ctx_static : opt_ctx->ctx_compute;385 386    switch (opt_ctx->loss_type) {387        case GGML_OPT_LOSS_TYPE_MEAN: {388            opt_ctx->loss = ggml_sum(ctx_results, opt_ctx->outputs);389            ggml_set_name(opt_ctx->loss, "loss_sum");390            const float scale = 1.0f / (opt_ctx->opt_period * ggml_nelements(opt_ctx->outputs));391            opt_ctx->loss = ggml_scale(ctx_results, opt_ctx->loss, scale);392            ggml_set_name(opt_ctx->loss, "loss_mean");393            opt_ctx->loss_per_datapoint = true;394            break;395        }396        case GGML_OPT_LOSS_TYPE_SUM: {397            opt_ctx->loss = ggml_sum(ctx_results, opt_ctx->outputs);398            ggml_set_name(opt_ctx->loss, "loss_sum");399            opt_ctx->loss_per_datapoint = false;400            break;401        }402        case GGML_OPT_LOSS_TYPE_CROSS_ENTROPY: {403            opt_ctx->labels = ggml_dup_tensor(ctx_results, opt_ctx->outputs);404            ggml_set_input(opt_ctx->labels);405            ggml_set_name(opt_ctx->labels, "labels");406            opt_ctx->loss = ggml_cross_entropy_loss(ctx_results, opt_ctx->outputs, opt_ctx->labels);407            ggml_set_name(opt_ctx->loss, "loss_cross_entropy");408            if (opt_ctx->opt_period > 1) {409                opt_ctx->loss = ggml_scale(ctx_results, opt_ctx->loss, 1.0f / opt_ctx->opt_period);410                ggml_set_name(opt_ctx->loss, "loss_cross_entropy_scaled");411            }412            opt_ctx->loss_per_datapoint = true;413            break;414        }415        case GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR: {416            opt_ctx->labels = ggml_dup_tensor(ctx_results, opt_ctx->outputs);417            ggml_set_input(opt_ctx->labels);418            ggml_set_name(opt_ctx->labels, "labels");419            opt_ctx->loss = ggml_sub(ctx_results, opt_ctx->outputs, opt_ctx->labels);420            ggml_set_name(opt_ctx->loss, "loss_error");421            opt_ctx->loss = ggml_sqr(ctx_results, opt_ctx->loss);422            ggml_set_name(opt_ctx->loss, "loss_squared_error");423            opt_ctx->loss = ggml_sum(ctx_results, opt_ctx->loss);424            ggml_set_name(opt_ctx->loss, "loss_sum_squared_error");425            const float scale = 1.0f / (opt_ctx->opt_period * ggml_nelements(opt_ctx->outputs));426            opt_ctx->loss = ggml_scale(ctx_results, opt_ctx->loss, scale);427            ggml_set_name(opt_ctx->loss, "loss_mean_squared_error");428            opt_ctx->loss_per_datapoint = true;429            break;430        }431    }432    ggml_set_output(opt_ctx->loss);433    ggml_set_loss(opt_ctx->loss);434    ggml_build_forward_expand(opt_ctx->gf, opt_ctx->loss);435 436    if (opt_ctx->loss_type == GGML_OPT_LOSS_TYPE_CROSS_ENTROPY) {437        opt_ctx->pred = ggml_argmax(ctx_results, opt_ctx->outputs);438        ggml_set_name(opt_ctx->pred, "pred");439        ggml_set_output(opt_ctx->pred);440        ggml_build_forward_expand(opt_ctx->gf, opt_ctx->pred);441 442        opt_ctx->ncorrect = ggml_count_equal(ctx_results, opt_ctx->pred, ggml_argmax(ctx_results, opt_ctx->labels));443        ggml_set_name(opt_ctx->ncorrect, "ncorrect");444        ggml_set_output(opt_ctx->ncorrect);445        ggml_build_forward_expand(opt_ctx->gf, opt_ctx->ncorrect);446    }447 448    if (opt_ctx->buf_static) {449        if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_FORWARD) {450            return;451        }452    } else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_FORWARD) {453        opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(454            opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));455        return;456    }457 458    if (opt_ctx->grad_accs.empty()) {459        GGML_ASSERT(opt_ctx->build_type_alloc >= GGML_OPT_BUILD_TYPE_GRAD);460 461        const int n_nodes = opt_ctx->gf->n_nodes;462        opt_ctx->grad_accs.resize(n_nodes);463        for (int i = 0; i < n_nodes; ++i) {464            ggml_tensor * node = opt_ctx->gf->nodes[i];465            if ((accumulate && (node->flags & GGML_TENSOR_FLAG_PARAM)) || (node->flags & GGML_TENSOR_FLAG_LOSS)) {466                opt_ctx->grad_accs[i] = ggml_new_tensor(opt_ctx->ctx_static, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);467            } else {468                opt_ctx->grad_accs[i] = nullptr;469            }470        }471 472        if (need_momenta && opt_ctx->build_type_alloc >= GGML_OPT_BUILD_TYPE_OPT) {473            opt_ctx->grad_m.resize(n_nodes);474            opt_ctx->grad_v.resize(n_nodes);475            for (int i = 0; i < n_nodes; ++i) {476                ggml_tensor * node = opt_ctx->gf->nodes[i];477                if (node->flags & GGML_TENSOR_FLAG_PARAM) {478                    opt_ctx->grad_m[i] = ggml_new_tensor(opt_ctx->ctx_static, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);479                    opt_ctx->grad_v[i] = ggml_new_tensor(opt_ctx->ctx_static, GGML_TYPE_F32, GGML_MAX_DIMS, node->ne);480                } else {481                    opt_ctx->grad_m[i] = nullptr;482                    opt_ctx->grad_v[i] = nullptr;483                }484            }485        }486    }487 488    // gb_grad == graph backward gradients, forward pass, then backward pass to calculate gradients.489    opt_ctx->gb_grad = ggml_graph_dup(opt_ctx->ctx_compute, opt_ctx->gf, /*force_grads =*/ true);490    ggml_build_backward_expand(opt_ctx->ctx_compute, opt_ctx->gb_grad, opt_ctx->grad_accs.data());491 492    if (opt_ctx->buf_static) {493        if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_GRAD) {494            return;495        }496    } else if (opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_GRAD) {497        opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));498        ggml_graph_reset(opt_ctx->gb_grad);499    }500 501    GGML_ASSERT(opt_ctx->build_type_alloc == GGML_OPT_BUILD_TYPE_OPT);502 503    // gb_opt == graph backward optimize, forward pass, then backward pass to calculate gradients, then optimizer step.504    opt_ctx->gb_opt = ggml_graph_dup(opt_ctx->ctx_compute, opt_ctx->gb_grad, /*force_grads =*/ true);505 506    opt_ctx->opt_step_params = ggml_new_tensor_1d(opt_ctx->ctx_cpu, GGML_TYPE_F32, need_momenta ? 7 : 2);507    ggml_tensor * adamw_params = opt_ctx->opt_step_params;508    ggml_set_input(adamw_params);509    const char * optimizer_name = ggml_opt_optimizer_name(opt_ctx->optimizer);510    ggml_format_name(adamw_params, "%s_params", optimizer_name);511    for (int i = opt_ctx->gf->n_nodes-1; i >= 0; --i) {512        struct ggml_tensor * node = opt_ctx->gb_opt->nodes[i];513        struct ggml_tensor * grad = ggml_graph_get_grad(opt_ctx->gb_opt, node);514 515        if (grad && (node->flags & GGML_TENSOR_FLAG_PARAM)) {516            struct ggml_tensor * m = nullptr;517            struct ggml_tensor * v = nullptr;518            if (need_momenta) {519                m = opt_ctx->grad_m[i];520                v = opt_ctx->grad_v[i];521                ggml_format_name(m, "AdamW m for %s", node->name);522                ggml_format_name(v, "AdamW v for %s", node->name);523            }524            struct ggml_tensor * opt_step;525            switch (optimizer) {526                case GGML_OPT_OPTIMIZER_TYPE_ADAMW:527                    opt_step = ggml_opt_step_adamw(opt_ctx->ctx_compute, node, grad, m, v, adamw_params);528                    break;529                case GGML_OPT_OPTIMIZER_TYPE_SGD:530                    opt_step = ggml_opt_step_sgd(opt_ctx->ctx_compute, node, grad, adamw_params);531                    break;532                default:533                    GGML_ABORT("fatal error");534            }535            ggml_format_name(opt_step, "%s step for %s", optimizer_name, node->name);536            ggml_build_forward_expand(opt_ctx->gb_opt, opt_step);537        }538    }539 540    if (!opt_ctx->buf_static) {541        opt_ctx->buf_static = ggml_backend_alloc_ctx_tensors(542            opt_ctx->ctx_static, ggml_backend_sched_get_backend(opt_ctx->backend_sched, 0));543        ggml_graph_reset(opt_ctx->gb_opt);544    }545 546    opt_ctx->buf_cpu = ggml_backend_alloc_ctx_tensors_from_buft(opt_ctx->ctx_cpu, ggml_backend_cpu_buffer_type());547}548 549ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params) {550    ggml_opt_context_t result = new struct ggml_opt_context;551    result->backend_sched    = params.backend_sched;552    result->ctx_compute      = params.ctx_compute;553    result->loss_type        = params.loss_type;554    result->build_type       = params.build_type;555    result->build_type_alloc = params.build_type;556    result->inputs           = params.inputs;557    result->outputs          = params.outputs;558    result->opt_period       = params.opt_period;559    result->get_opt_pars     = params.get_opt_pars;560    result->get_opt_pars_ud  = params.get_opt_pars_ud;561    result->optimizer        = params.optimizer;562 563    GGML_ASSERT(result->opt_period >= 1);564 565    result->static_graphs = result->ctx_compute;566 567    if (!result->static_graphs) {568        GGML_ASSERT(!result->inputs);569        GGML_ASSERT(!result->outputs);570        return result;571    }572 573    GGML_ASSERT(result->inputs);574    GGML_ASSERT(result->outputs);575 576    result->gf = ggml_new_graph_custom(result->ctx_compute, GGML_DEFAULT_GRAPH_SIZE, /*grads =*/ true); // Forward pass.577    ggml_build_forward_expand(result->gf, result->outputs);578 579    ggml_opt_build(result);580 581    return result;582}583 584void ggml_opt_free(ggml_opt_context_t opt_ctx) {585    if (opt_ctx == nullptr) {586        return;587    }588    ggml_backend_buffer_free(opt_ctx->buf_static);589    ggml_backend_buffer_free(opt_ctx->buf_cpu);590    ggml_free(opt_ctx->ctx_static);591    ggml_free(opt_ctx->ctx_cpu);592    ggml_free(opt_ctx->ctx_copy);593    delete opt_ctx;594}595 596void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer) {597    if (optimizer) {598        ggml_graph_reset(opt_ctx->gb_opt);599        opt_ctx->iter = 1;600    } else {601        ggml_graph_reset(opt_ctx->gb_grad);602    }603}604 605bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx) {606    return opt_ctx->static_graphs;607}608 609struct ggml_tensor * ggml_opt_inputs(ggml_opt_context_t opt_ctx) {610    return opt_ctx->inputs;611}612 613struct ggml_tensor * ggml_opt_outputs(ggml_opt_context_t opt_ctx) {614    return opt_ctx->outputs;615}616 617struct ggml_tensor * ggml_opt_labels(ggml_opt_context_t opt_ctx) {618    return opt_ctx->labels;619}620 621struct ggml_tensor * ggml_opt_loss(ggml_opt_context_t opt_ctx) {622    return opt_ctx->loss;623}624 625struct ggml_tensor * ggml_opt_pred(ggml_opt_context_t opt_ctx) {626    return opt_ctx->pred;627}628 629struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx) {630    return opt_ctx->ncorrect;631}632 633struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node) {634    return ggml_graph_get_grad_acc(opt_ctx->gb_opt, node);635}636 637// ====== Optimization Result ======638 639ggml_opt_result_t ggml_opt_result_init() {640    return new ggml_opt_result;641}642 643void ggml_opt_result_free(ggml_opt_result_t result) {644    delete result;645}646 647void ggml_opt_result_reset(ggml_opt_result_t result) {648    result->ndata = 0;649    result->loss.clear();650    result->pred.clear();651    result->ncorrect = 0;652}653 654void ggml_opt_result_ndata(ggml_opt_result_t result, int64_t * ndata) {655    *ndata = result->ndata;656}657 658void ggml_opt_result_loss(ggml_opt_result_t result, double * loss, double * unc) {659    const int64_t nbatches = result->loss.size(); // Number of physical batches.660 661    if (nbatches == 0) {662        *loss = 0.0;663        *unc  = NAN;664        return;665    }666 667    double sum         = 0.0;668    double sum_squared = 0.0;669 670    for (const float & loss : result->loss) {671        // If the loss is per datapoint it was scaled by 1.0f/opt_period for each physical batch.672        const float loss_scaled = result->loss_per_datapoint ? loss*result->opt_period : loss;673        sum         += loss_scaled;674        sum_squared += loss_scaled*loss_scaled;675    }676 677    const double mean = sum/nbatches;678    *loss = result->loss_per_datapoint ? mean : sum;679 680    if (!unc) {681        return;682    }683 684    if (nbatches < 2) {685        *unc = NAN;686        return;687    }688 689    const double var_sum = sum_squared/nbatches - mean*mean; // variance without Bessel's correction, i.e. nbatches/(nbatches-1)690    *unc = result->loss_per_datapoint ? sqrt(var_sum / (nbatches - 1)) : sqrt(var_sum * nbatches/(nbatches - 1));691}692 693void ggml_opt_result_pred(ggml_opt_result_t result, int32_t * pred) {694    for (size_t i = 0; i < result->pred.size(); ++i) {695        pred[i] = result->pred[i];696    }697}698 699void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc) {700    *accuracy = result->ncorrect >= 0 ? double(result->ncorrect) / double(result->ndata) : NAN;701 702    if (!unc) {703        return;704    }705 706    *unc = result->ncorrect >= 0 && result->ndata >= 2 ?707        sqrt((*accuracy) * (1.0 - (*accuracy)) / double(result->ndata - 1)) : NAN;708}709 710// ====== Computation ======711 712void ggml_opt_prepare_alloc(713        ggml_opt_context_t    opt_ctx,714        struct ggml_context * ctx_compute,715        struct ggml_cgraph  * gf,716        struct ggml_tensor  * inputs,717        struct ggml_tensor  * outputs) {718    GGML_ASSERT(!opt_ctx->static_graphs);719    opt_ctx->ctx_compute = ctx_compute;720    opt_ctx->gf          = gf;721    opt_ctx->inputs      = inputs;722    opt_ctx->outputs     = outputs;723}724 725void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward) {726    GGML_ASSERT(!opt_ctx->eval_ready);727    if (opt_ctx->build_type == GGML_OPT_BUILD_TYPE_OPT && opt_ctx->opt_period > 1 && opt_ctx->opt_i == 0) {728        ggml_graph_reset(opt_ctx->gb_grad);729    }730    if (backward) {731        const int32_t opt_i_next = (opt_ctx->opt_i + 1) % opt_ctx->opt_period;732        opt_ctx->build_type = opt_i_next == 0 ? GGML_OPT_BUILD_TYPE_OPT : GGML_OPT_BUILD_TYPE_GRAD;733    } else {734        opt_ctx->build_type = GGML_OPT_BUILD_TYPE_FORWARD;735    }736 737    if (!opt_ctx->static_graphs) {738        ggml_opt_build(opt_ctx);739    }740 741    struct ggml_cgraph * graph = nullptr;742    switch (opt_ctx->build_type) {743        case GGML_OPT_BUILD_TYPE_FORWARD: {744            graph = opt_ctx->gf;745        } break;746        case GGML_OPT_BUILD_TYPE_GRAD: {747            graph = opt_ctx->gb_grad;748        } break;749        case GGML_OPT_BUILD_TYPE_OPT: {750            graph = opt_ctx->gb_opt;751        } break;752    }753    GGML_ASSERT(graph);754 755    if (opt_ctx->allocated_graph == graph) {756        opt_ctx->eval_ready = true;757        return;758    }759 760    ggml_backend_sched_reset(opt_ctx->backend_sched); // clear allocation of previous graph761 762    if (opt_ctx->static_graphs) {763        ggml_init_params params = {764            /*.mem_size   =*/ graph->size*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph->size, graph->grads),765            /*.mem_buffer =*/ nullptr,766            /*.no_alloc   =*/ true,767        };768        ggml_free(opt_ctx->ctx_copy);769        opt_ctx->ctx_copy = ggml_init(params);770 771        opt_ctx->allocated_graph_copy = dup_graph(opt_ctx->ctx_copy, graph);772    } else {773        opt_ctx->allocated_graph_copy = graph;774    }775 776    ggml_backend_sched_alloc_graph(opt_ctx->backend_sched, opt_ctx->allocated_graph_copy);777    opt_ctx->allocated_graph = graph;778 779    opt_ctx->eval_ready = true;780}781 782void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result) {783    GGML_ASSERT(opt_ctx->eval_ready);784    if (opt_ctx->allocated_graph == opt_ctx->gb_opt) {785        const ggml_opt_optimizer_params & opt_pars = opt_ctx->get_opt_pars(opt_ctx->get_opt_pars_ud);786 787        switch (opt_ctx->optimizer) {788            case GGML_OPT_OPTIMIZER_TYPE_ADAMW: {789                GGML_ASSERT(opt_pars.adamw.alpha > 0.0f);790                GGML_ASSERT(opt_pars.adamw.beta1 >= 0.0f);791                GGML_ASSERT(opt_pars.adamw.beta1 <= 1.0f);792                GGML_ASSERT(opt_pars.adamw.beta2 >= 0.0f);793                GGML_ASSERT(opt_pars.adamw.beta2 <= 1.0f);794                GGML_ASSERT(opt_pars.adamw.eps >= 0.0f);795                GGML_ASSERT(opt_pars.adamw.wd >= 0.0f);796                GGML_ASSERT(opt_pars.adamw.wd <= 1.0f);797 798                // beta1, beta2 after applying warmup799                const float beta1h = 1.0f / (1.0f - powf(opt_pars.adamw.beta1, opt_ctx->iter));800                const float beta2h = 1.0f / (1.0f - powf(opt_pars.adamw.beta2, opt_ctx->iter));801 802                float * adamw_par_data = ggml_get_data_f32(opt_ctx->opt_step_params);803                adamw_par_data[0] = opt_pars.adamw.alpha;804                adamw_par_data[1] = opt_pars.adamw.beta1;805                adamw_par_data[2] = opt_pars.adamw.beta2;806                adamw_par_data[3] = opt_pars.adamw.eps;807                adamw_par_data[4] = opt_pars.adamw.wd;808                adamw_par_data[5] = beta1h;809                adamw_par_data[6] = beta2h;810            } break;811            case GGML_OPT_OPTIMIZER_TYPE_SGD: {812                GGML_ASSERT(opt_pars.sgd.alpha > 0.0f);813                GGML_ASSERT(opt_pars.sgd.wd >= 0.0f);814                GGML_ASSERT(opt_pars.sgd.wd <= 1.0f);815                float * sgd = ggml_get_data_f32(opt_ctx->opt_step_params);816                sgd[0] = opt_pars.sgd.alpha;817                sgd[1] = opt_pars.sgd.wd;818            } break;819            default:820                GGML_ABORT("fatal error");821        }822    }823 824    ggml_backend_sched_graph_compute(opt_ctx->backend_sched, opt_ctx->allocated_graph_copy);825    opt_ctx->iter += opt_ctx->allocated_graph == opt_ctx->gb_opt;826    opt_ctx->opt_i = (opt_ctx->opt_i + 1) % opt_ctx->opt_period;827 828    if (!opt_ctx->static_graphs) {829        opt_ctx->gf                   = nullptr;830        opt_ctx->gb_grad              = nullptr;831        opt_ctx->gb_opt               = nullptr;832        opt_ctx->allocated_graph      = nullptr;833        opt_ctx->allocated_graph_copy = nullptr;834    }835 836    opt_ctx->eval_ready = false;837 838    if (!result) {839        return;840    }841 842    if (result->ndata == 0) {843        result->loss_per_datapoint = opt_ctx->loss_per_datapoint;844        result->opt_period         = opt_ctx->opt_period;845    } else {846        GGML_ASSERT(result->loss_per_datapoint == opt_ctx->loss_per_datapoint);847        GGML_ASSERT(result->opt_period         == opt_ctx->opt_period);848    }849 850    const int64_t ndata = opt_ctx->outputs->ne[1];851    GGML_ASSERT(result->ndata == ndata*int64_t(result->loss.size()) && "varying batch size not supported");852    result->ndata += ndata;853 854    GGML_ASSERT(ggml_is_scalar(opt_ctx->loss));855    GGML_ASSERT(opt_ctx->loss->type == GGML_TYPE_F32);856    float loss;857    ggml_backend_tensor_get(opt_ctx->loss, &loss, 0, ggml_nbytes(opt_ctx->loss));858    result->loss.push_back(loss);859 860    if (opt_ctx->pred) {861        GGML_ASSERT(opt_ctx->pred->type == GGML_TYPE_I32);862        std::vector<int32_t> pred(ndata);863        ggml_backend_tensor_get(opt_ctx->pred, pred.data(), 0, ggml_nbytes(opt_ctx->pred));864        result->pred.insert(result->pred.end(), pred.begin(), pred.end());865    }866 867    if (!opt_ctx->ncorrect || result->ncorrect < 0) {868        result->ncorrect = -1;869        return;870    }871 872    GGML_ASSERT(ggml_is_scalar(opt_ctx->ncorrect));873    GGML_ASSERT(opt_ctx->ncorrect->type == GGML_TYPE_I64);874    int64_t ncorrect;875    ggml_backend_tensor_get(opt_ctx->ncorrect, &ncorrect, 0, ggml_nbytes(opt_ctx->ncorrect));876    result->ncorrect += ncorrect;877}878 879// ====== High-Level Functions ======880 881void ggml_opt_epoch(882        ggml_opt_context_t      opt_ctx,883        ggml_opt_dataset_t      dataset,884        ggml_opt_result_t       result_train,885        ggml_opt_result_t       result_eval,886        int64_t                 idata_split,887        ggml_opt_epoch_callback callback_train,888        ggml_opt_epoch_callback callback_eval) {889    GGML_ASSERT(ggml_opt_static_graphs(opt_ctx) && "ggml_opt_epoch requires static graphs");890    struct ggml_tensor * inputs = ggml_opt_inputs(opt_ctx);891    struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);892    struct ggml_tensor * data   = ggml_opt_dataset_data(dataset);893    GGML_ASSERT(data->ne[0] == inputs->ne[0]);894 895    const int64_t ndata       =   data->ne[1];896    const int64_t ndata_batch = inputs->ne[1];897 898    GGML_ASSERT(data->ne[1] % inputs->ne[1] == 0);899    const int64_t nbatches = ndata/ndata_batch;900 901    idata_split = idata_split < 0 ? ndata : idata_split;902    GGML_ASSERT(idata_split % ndata_batch == 0);903    const int64_t ibatch_split = idata_split / ndata_batch;904 905    int64_t ibatch = 0;906    int64_t t_loop_start = ggml_time_us();907    for (; ibatch < ibatch_split; ++ibatch) {908        ggml_opt_alloc(opt_ctx, /*backward =*/ true);909        ggml_opt_dataset_get_batch(dataset, inputs, labels, ibatch);910        ggml_opt_eval(opt_ctx, result_train);911        if (callback_train) {912            callback_train(true, opt_ctx, dataset, result_train, ibatch+1, ibatch_split, t_loop_start);913        }914    }915    t_loop_start = ggml_time_us();916    for (; ibatch < nbatches; ++ibatch) {917        ggml_opt_alloc(opt_ctx, /*backward =*/ false);918        ggml_opt_dataset_get_batch(dataset, inputs, labels, ibatch);919        ggml_opt_eval(opt_ctx, result_eval);920        if (callback_eval) {921            callback_eval(false, opt_ctx, dataset, result_eval, ibatch+1-ibatch_split, nbatches-ibatch_split, t_loop_start);922        }923    }924}925 926void ggml_opt_epoch_callback_progress_bar(927        bool               train,928        ggml_opt_context_t opt_ctx,929        ggml_opt_dataset_t dataset,930        ggml_opt_result_t  result,931        int64_t            ibatch,932        int64_t            ibatch_max,933        int64_t            t_start_us) {934    fprintf(stderr, "%s[", train ? "train: " : "val:   ");935 936    // The progress bar consists of partially filled blocks, unicode has 8 separate fill levels.937    constexpr int64_t bar_length = 8;938    const int64_t ibatch8 = 8 * ibatch;939    for (int64_t j = 0; j < bar_length; ++j) {940        if        (ibatch_max * (8*j + 8) / bar_length < ibatch8) {941            fprintf(stderr, "\u2588"); // full block942        } else if (ibatch_max * (8*j + 7) / bar_length < ibatch8) {943            fprintf(stderr, "\u2589"); // 7/8 filled944        } else if (ibatch_max * (8*j + 6) / bar_length < ibatch8) {945            fprintf(stderr, "\u258A"); // 6/8 filled946        } else if (ibatch_max * (8*j + 5) / bar_length < ibatch8) {947            fprintf(stderr, "\u258B"); // 5/8 filled948        } else if (ibatch_max * (8*j + 4) / bar_length < ibatch8) {949            fprintf(stderr, "\u258C"); // 4/8 filled950        } else if (ibatch_max * (8*j + 3) / bar_length < ibatch8) {951            fprintf(stderr, "\u258D"); // 3/8 filled952        } else if (ibatch_max * (8*j + 2) / bar_length < ibatch8) {953            fprintf(stderr, "\u258E"); // 2/8 filled954        } else if (ibatch_max * (8*j + 1) / bar_length < ibatch8) {955            fprintf(stderr, "\u258F"); // 1/8 filled956        } else {957            fprintf(stderr, " ");958        }959    }960 961    const int64_t batch_size = ggml_opt_inputs(opt_ctx)->ne[1];962    const int64_t idata      = ibatch*batch_size;963    const int64_t idata_max  = ibatch_max*batch_size;964 965    double loss;966    double loss_unc;967    ggml_opt_result_loss(result, &loss, &loss_unc);968 969    double accuracy;970    double accuracy_unc;971    ggml_opt_result_accuracy(result, &accuracy, &accuracy_unc);972 973    const int64_t t_ibatch_us = ggml_time_us() - t_start_us;974    int64_t t_ibatch_s = t_ibatch_us / 1000000;975    const int64_t t_ibatch_h = t_ibatch_s / 3600;976    t_ibatch_s -= t_ibatch_h * 3600;977    const int64_t t_ibatch_m = t_ibatch_s / 60;978    t_ibatch_s -= t_ibatch_m * 60;979 980    const int64_t t_eta_us = t_ibatch_us * (ibatch_max - ibatch)/ibatch;981    int64_t t_eta_s = t_eta_us / 1000000;982    const int64_t t_eta_h = t_eta_s / 3600;983    t_eta_s -= t_eta_h * 3600;984    const int64_t t_eta_m = t_eta_s / 60;985    t_eta_s -= t_eta_m * 60;986 987    fprintf(stderr, "] data=%07" PRId64 "/%07" PRId64 " loss=%.5lf±%.5lf acc=%.2lf±%.2lf%% "988            "t=%02" PRId64 ":%02" PRId64 ":%02" PRId64 " ETA=%02" PRId64 ":%02" PRId64 ":%02" PRId64 " \r",989            idata, idata_max, loss, loss_unc, 100.0*accuracy, 100.0*accuracy_unc,990            t_ibatch_h, t_ibatch_m, t_ibatch_s, t_eta_h, t_eta_m, t_eta_s);991    if (ibatch == ibatch_max) {992        fprintf(stderr, "\n");993    }994    fflush(stderr);995 996    GGML_UNUSED(dataset);997}998 999void ggml_opt_fit(1000        ggml_backend_sched_t            backend_sched,1001        ggml_context                  * ctx_compute,1002        ggml_tensor                   * inputs,1003        ggml_tensor                   * outputs,1004        ggml_opt_dataset_t              dataset,1005        enum ggml_opt_loss_type         loss_type,1006        enum ggml_opt_optimizer_type    optimizer,1007        ggml_opt_get_optimizer_params   get_opt_pars,1008        int64_t                         nepoch,1009        int64_t                         nbatch_logical,1010        float                           val_split,1011        bool                            silent) {1012    ggml_time_init();1013    const int64_t t_start_us = ggml_time_us();1014 1015    const int64_t ndata           = ggml_opt_dataset_data(dataset)->ne[1];1016    const int64_t nbatch_physical = inputs->ne[1];1017    GGML_ASSERT(ndata          % nbatch_logical  == 0);1018    GGML_ASSERT(nbatch_logical % nbatch_physical == 0);1019 1020    const int64_t opt_period       = nbatch_logical / nbatch_physical;1021    const int64_t nbatches_logical = ndata / nbatch_logical;1022 1023    GGML_ASSERT(val_split >= 0.0f);1024    GGML_ASSERT(val_split <  1.0f);1025    const int64_t ibatch_split = int64_t(((1.0f - val_split) * nbatches_logical)) * opt_period; // train <-> val split index (physical)1026    const int64_t idata_split  = ibatch_split * nbatch_physical;1027 1028    int64_t epoch = 1;1029 1030    ggml_opt_params params = ggml_opt_default_params(backend_sched, loss_type);1031    params.ctx_compute     = ctx_compute;1032    params.inputs          = inputs;1033    params.outputs         = outputs;1034    params.opt_period      = opt_period;1035    params.get_opt_pars    = get_opt_pars;1036    params.get_opt_pars_ud = &epoch;1037    params.optimizer       = optimizer;1038    ggml_opt_context_t opt_ctx = ggml_opt_init(params);1039 1040    // Shuffling the data is generally useful but there is only a point if not all data is used in a single batch.1041    if (nbatch_logical < ndata) {1042        ggml_opt_dataset_shuffle(opt_ctx, dataset, -1); // Shuffle all data (train + validation).1043    }1044 1045    ggml_opt_result_t result_train = ggml_opt_result_init();1046    ggml_opt_result_t result_val   = ggml_opt_result_init();1047 1048    ggml_opt_epoch_callback epoch_callback = silent ? nullptr : ggml_opt_epoch_callback_progress_bar;1049 1050    for (; epoch <= nepoch; ++epoch) {1051        if (nbatch_logical < idata_split) {1052            ggml_opt_dataset_shuffle(opt_ctx, dataset, idata_split);1053        }1054 1055        ggml_opt_result_reset(result_train);1056        ggml_opt_result_reset(result_val);1057 1058        if (!silent) {1059            fprintf(stderr, "%s: epoch %04" PRId64 "/%04" PRId64 ":\n", __func__, epoch, nepoch);1060        }1061        ggml_opt_epoch(opt_ctx, dataset, result_train, result_val, idata_split, epoch_callback, epoch_callback);1062        if (!silent) {1063            fprintf(stderr, "\n");1064        }1065    }1066 1067    if (!silent) {1068        int64_t t_total_s = (ggml_time_us() - t_start_us) / 1000000;1069        const int64_t t_total_h = t_total_s / 3600;1070        t_total_s -= t_total_h * 3600;1071        const int64_t t_total_m = t_total_s / 60;1072        t_total_s -= t_total_m * 60;1073        fprintf(stderr, "%s: training took %02" PRId64 ":%02" PRId64 ":%02" PRId64 "\n", __func__, t_total_h, t_total_m, t_total_s);1074    }1075 1076    ggml_opt_free(opt_ctx);1077    ggml_opt_result_free(result_train);1078    ggml_opt_result_free(result_val);1079}1080 1081enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t c) {1082    return c->optimizer;1083}1084 1085GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type o) {1086    switch (o) {1087        case GGML_OPT_OPTIMIZER_TYPE_ADAMW:1088            return "adamw";1089        case GGML_OPT_OPTIMIZER_TYPE_SGD:1090            return "sgd";1091        default:1092            return "undefined";1093    };1094}1095