KBaba7/llama.cpp

1#!/bin/bash2 3set -eu4 5if [ $# -lt 1 ]6then7    echo "usage:   $0 path_to_build_binary [path_to_temp_folder]"8    echo "example: $0 ../../build/bin ../../tmp"9    exit 110fi11 12if [ $# -gt 1 ]13then14    TMP_DIR=$215else16    TMP_DIR=/tmp17fi18 19set -x20 21SPLIT=$1/llama-gguf-split22QUANTIZE=$1/llama-quantize23MAIN=$1/llama-cli24WORK_PATH=$TMP_DIR/quantize25ROOT_DIR=$(realpath $(dirname $0)/../../)26 27mkdir -p "$WORK_PATH"28 29# Clean up in case of previously failed test30rm -f $WORK_PATH/ggml-model-split*.gguf $WORK_PATH/ggml-model-requant*.gguf31 32# 1. Get a model33(34cd $WORK_PATH35"$ROOT_DIR"/scripts/hf.sh --repo ggml-org/gemma-1.1-2b-it-Q8_0-GGUF --file gemma-1.1-2b-it.Q8_0.gguf36)37echo PASS38 39# 2. Split model40$SPLIT --split-max-tensors 28  $WORK_PATH/gemma-1.1-2b-it.Q8_0.gguf $WORK_PATH/ggml-model-split41echo PASS42echo43 44# 3. Requant model with '--keep-split'45$QUANTIZE --allow-requantize --keep-split $WORK_PATH/ggml-model-split-00001-of-00006.gguf $WORK_PATH/ggml-model-requant.gguf Q4_K46echo PASS47echo48 49# 3a. Test the requanted model is loading properly50$MAIN -no-cnv --model $WORK_PATH/ggml-model-requant-00001-of-00006.gguf --n-predict 3251echo PASS52echo53 54# 4. Requant mode without '--keep-split'55$QUANTIZE --allow-requantize $WORK_PATH/ggml-model-split-00001-of-00006.gguf $WORK_PATH/ggml-model-requant-merge.gguf Q4_K56echo PASS57echo58 59# 4b. Test the requanted model is loading properly60$MAIN -no-cnv --model $WORK_PATH/ggml-model-requant-merge.gguf --n-predict 3261echo PASS62echo63 64# Clean up65rm -f $WORK_PATH/ggml-model-split*.gguf $WORK_PATH/ggml-model-requant*.gguf66