KBaba7/llama.cpp
0
1// This file contains functionality for training models using GGML.2// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets.3// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code.4//5// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de)6 7#pragma once8 9#include "ggml.h"10#include "ggml-backend.h"11 12#include <stdint.h>13 14#ifdef __cplusplus15extern "C" {16#endif17 18 struct ggml_opt_dataset;19 struct ggml_opt_context;20 struct ggml_opt_result;21 22 typedef struct ggml_opt_dataset * ggml_opt_dataset_t;23 typedef struct ggml_opt_context * ggml_opt_context_t;24 typedef struct ggml_opt_result * ggml_opt_result_t;25 26 // ====== Loss ======27 28 // built-in loss types, i.e. the built-in quantities minimized by the optimizer29 // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value30 enum ggml_opt_loss_type {31 GGML_OPT_LOSS_TYPE_MEAN,32 GGML_OPT_LOSS_TYPE_SUM,33 GGML_OPT_LOSS_TYPE_CROSS_ENTROPY,34 GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR,35 };36 37 // ====== Dataset ======38 39 GGML_API ggml_opt_dataset_t ggml_opt_dataset_init(40 int64_t ne_datapoint, // number of elements per datapoint41 int64_t ne_label, // number of elements per label42 int64_t ndata, // total number of datapoints/labels43 int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied)44 GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset);45 46 // get underlying tensors that store the data47 GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata]48 GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata]49 50 // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative51 GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata);52 53 // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch54 GGML_API void ggml_opt_dataset_get_batch(55 ggml_opt_dataset_t dataset,56 struct ggml_tensor * data_batch, // shape = [ne_datapoint, ndata_batch]57 struct ggml_tensor * labels_batch, // shape = [ne_label, ndata_batch]58 int64_t ibatch);59 60 // ====== Model / Context ======61 62 enum ggml_opt_build_type {63 GGML_OPT_BUILD_TYPE_FORWARD,64 GGML_OPT_BUILD_TYPE_GRAD,65 GGML_OPT_BUILD_TYPE_OPT,66 };67 68 // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss69 struct ggml_opt_optimizer_params {70 // AdamW optimizer parameters71 struct {72 float alpha; // learning rate73 float beta1;74 float beta2;75 float eps; // epsilon for numerical stability76 float wd; // weight decay for AdamW, use 0.0f to disable77 } adamw;78 };79 80 // callback to calculate optimizer parameters prior to a backward pass81 // userdata can be used to pass arbitrary data82 typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata);83 84 // returns the default optimizer params (constant)85 // userdata is not used86 GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata);87 88 // parameters for initializing a new optimization context89 struct ggml_opt_params {90 ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs91 92 struct ggml_context * ctx_compute; // created in user code, holds non-static tensors93 94 // the forward graph is defined by inputs and outputs95 // those tensors and all tensors inbetween are not intended to be reusable between multiple optimization contexts96 struct ggml_tensor * inputs;97 struct ggml_tensor * outputs;98 99 enum ggml_opt_loss_type loss_type;100 enum ggml_opt_build_type build_type;101 102 int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done103 104 ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters105 void * get_opt_pars_ud; // userdata for calculating optimizer parameters106 };107 108 // get parameters for an optimization context with defaults set where possible109 // parameters for which no sensible defaults exist are supplied as arguments to this function110 GGML_API ggml_opt_params ggml_opt_default_params(111 ggml_backend_sched_t backend_sched,112 struct ggml_context * ctx_compute,113 struct ggml_tensor * inputs,114 struct ggml_tensor * outputs,115 enum ggml_opt_loss_type loss_type);116 117 GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params);118 GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx);119 120 // set gradients to zero, initilize loss, and optionally reset the optimizer121 GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer);122 123 // get underlying tensors that store data124 GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor125 GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor126 GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against127 GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss128 GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs129 GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels130 131 GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node);132 133 // ====== Optimization Result ======134 135 GGML_API ggml_opt_result_t ggml_opt_result_init();136 GGML_API void ggml_opt_result_free(ggml_opt_result_t result);137 GGML_API void ggml_opt_result_reset(ggml_opt_result_t result);138 139 // get data from result, uncertainties are optional and can be ignored by passing NULL140 GGML_API void ggml_opt_result_ndata( ggml_opt_result_t result, int64_t * ndata); // writes 1 value, number of datapoints141 GGML_API void ggml_opt_result_loss( ggml_opt_result_t result, double * loss, double * unc); // writes 1 value142 GGML_API void ggml_opt_result_pred( ggml_opt_result_t result, int32_t * pred); // writes ndata values143 GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc); // writes 1 value144 145 // ====== Computation ======146 147 // do forward pass, increment result if not NULL148 GGML_API void ggml_opt_forward(ggml_opt_context_t opt_ctx, ggml_opt_result_t result);149 150 // do forward pass, increment result if not NULL, do backward pass151 GGML_API void ggml_opt_forward_backward(ggml_opt_context_t opt_ctx, ggml_opt_result_t result);152 153 // ############################################################################154 // ## The high-level functions start here. They do not depend on any private ##155 // ## functions or structs and can be copied to and adapted for user code. ##156 // ############################################################################157 158 // ====== Intended Usage ======159 //160 // 1. Select the appropriate loss for your problem.161 // 2. Create a dataset and set the data for the "data" tensor. Also set the "labels" tensor if your loss needs them.162 // Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster).163 // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors.164 // The first context should contain the model parameters and inputs and be allocated statically in user code.165 // The second context should contain all other tensors and will be (re)allocated automatically.166 // Due to this automated allocation the data of the second context is not defined when accessed in user code.167 // Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors.168 // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead.169 170 // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation171 typedef void (*ggml_opt_epoch_callback)(172 bool train, // true after training evaluation, false after validation evaluation173 ggml_opt_context_t opt_ctx,174 ggml_opt_dataset_t dataset,175 ggml_opt_result_t result, // result associated with the dataset subsection176 int64_t ibatch, // number of batches that have been evaluated so far177 int64_t ibatch_max, // total number of batches in this dataset subsection178 int64_t t_start_us); // time at which the evaluation on the dataset subsection was started179 180 // do training on front of dataset, do evaluation only on back of dataset181 GGML_API void ggml_opt_epoch(182 ggml_opt_context_t opt_ctx,183 ggml_opt_dataset_t dataset,184 ggml_opt_result_t result_train, // result to increment during training, ignored if NULL185 ggml_opt_result_t result_eval, // result to increment during evaluation, ignored if NULL186 int64_t idata_split, // data index at which to split training and evaluation187 ggml_opt_epoch_callback callback_train,188 ggml_opt_epoch_callback callback_eval);189 190 // callback that prints a progress bar on stderr191 GGML_API void ggml_opt_epoch_callback_progress_bar(192 bool train,193 ggml_opt_context_t opt_ctx,194 ggml_opt_dataset_t dataset,195 ggml_opt_result_t result,196 int64_t ibatch,197 int64_t ibatch_max,198 int64_t t_start_us);199 200 // fit model defined by inputs and outputs to dataset201 GGML_API void ggml_opt_fit(202 ggml_backend_sched_t backend_sched, // backend scheduler for constructing the compute graphs203 ggml_context * ctx_compute, // context with temporarily allocated tensors to calculate the outputs204 ggml_tensor * inputs, // input tensor with shape [ne_datapoint, ndata_batch]205 ggml_tensor * outputs, // output tensor, must have shape [ne_label, ndata_batch] if labels are used206 ggml_opt_dataset_t dataset, // dataset with data and optionally also labels207 enum ggml_opt_loss_type loss_type, // loss to minimize208 ggml_opt_get_optimizer_params get_opt_pars, // callback to get optimizer params, userdata is pointer to epoch (of type int64_t)209 int64_t nepoch, // how many times the dataset should be iterated over210 int64_t nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs211 float val_split, // fraction of the dataset to use for validation, must be in [0.0f, 1.0f)212 bool silent); // whether or not info prints to stderr should be suppressed213 214#ifdef __cplusplus215}216#endif217