KBaba7/llama.cpp

1#!/bin/bash2 3qnt=(q8_0 q6_k q5_k q5_1 q5_0 q4_k q4_1 q4_0 q3_k q2_k)4args=""5 6if [ -z "$1" ]; then7    echo "usage: $0 <model> [qnt] [args]"8    echo "default: $0 <model> \"${qnt[@]}\" \"${args}\""9    exit 110fi11 12if [ ! -z "$2" ]; then13    qnt=($2)14fi15 16if [ ! -z "$3" ]; then17    args="$3"18fi19 20model="$1"21out="../tmp/results-${model}"22 23set -o pipefail24set -e25 26mkdir -p ${out}27 28for q in ${qnt[@]}; do29    time ./bin/llama-quantize ../models/${model}/ggml-model-f16.gguf ../models/${model}/ggml-model-${q}.gguf ${q} 2>&1 ${args} | tee ${out}/qnt-${q}.txt30done31