KBaba7/llama.cpp

1#!/bin/bash2 3set -eu4 5if [ $# -lt 1 ]6then7    echo "usage:   $0 path_to_build_binary [path_to_temp_folder]"8    echo "example: $0 ../../build/bin ../../tmp"9    exit 110fi11 12if [ $# -gt 1 ]13then14    TMP_DIR=$215else16    TMP_DIR=/tmp17fi18 19set -x20 21SPLIT=$1/llama-gguf-split22MAIN=$1/llama-cli23WORK_PATH=$TMP_DIR/gguf-split24ROOT_DIR=$(realpath $(dirname $0)/../../)25 26mkdir -p "$WORK_PATH"27 28# Clean up in case of previously failed test29rm -f $WORK_PATH/ggml-model-split*.gguf $WORK_PATH/ggml-model-merge*.gguf30 31# 1. Get a model32(33cd $WORK_PATH34"$ROOT_DIR"/scripts/hf.sh --repo ggml-org/gemma-1.1-2b-it-Q8_0-GGUF --file gemma-1.1-2b-it.Q8_0.gguf35)36echo PASS37 38# 2. Split with max tensors strategy39$SPLIT --split-max-tensors 28  $WORK_PATH/gemma-1.1-2b-it.Q8_0.gguf $WORK_PATH/ggml-model-split40echo PASS41echo42 43# 2b. Test the sharded model is loading properly44$MAIN -no-cnv --model $WORK_PATH/ggml-model-split-00001-of-00006.gguf --n-predict 3245echo PASS46echo47 48# 3. Merge49$SPLIT --merge $WORK_PATH/ggml-model-split-00001-of-00006.gguf $WORK_PATH/ggml-model-merge.gguf50echo PASS51echo52 53# 3b. Test the merged model is loading properly54$MAIN -no-cnv --model $WORK_PATH/ggml-model-merge.gguf --n-predict 3255echo PASS56echo57 58# 4. Split with no tensors in the first split59$SPLIT --split-max-tensors 32 --no-tensor-first-split $WORK_PATH/ggml-model-merge.gguf $WORK_PATH/ggml-model-split-32-tensors60echo PASS61echo62 63# 4b. Test the sharded model is loading properly64$MAIN -no-cnv --model $WORK_PATH/ggml-model-split-32-tensors-00001-of-00007.gguf --n-predict 3265echo PASS66echo67 68# 5. Merge69#$SPLIT --merge $WORK_PATH/ggml-model-split-32-tensors-00001-of-00006.gguf $WORK_PATH/ggml-model-merge-2.gguf70#echo PASS71#echo72 73# 5b. Test the merged model is loading properly74#$MAIN -no-cnv --model $WORK_PATH/ggml-model-merge-2.gguf --n-predict 3275#echo PASS76#echo77 78# 6. Split with size strategy79$SPLIT --split-max-size 2G $WORK_PATH/ggml-model-merge.gguf $WORK_PATH/ggml-model-split-2G80echo PASS81echo82 83# 6b. Test the sharded model is loading properly84$MAIN -no-cnv --model $WORK_PATH/ggml-model-split-2G-00001-of-00002.gguf --n-predict 3285echo PASS86echo87 88# Clean up89rm -f $WORK_PATH/ggml-model-split*.gguf $WORK_PATH/ggml-model-merge*.gguf90