KBaba7/llama.cpp

1#!/bin/bash2 3qnt=(f16 q8_0 q6_k q5_k q5_1 q5_0 q4_k q4_1 q4_0 q3_k q2_k)4args="-ngl 999 -n 64 -p 512"5 6if [ -z "$1" ]; then7    echo "usage: $0 <model> [qnt] [args]"8    echo "default: $0 <model> \"${qnt[@]}\" \"${args}\""9    exit 110fi11 12if [ ! -z "$2" ]; then13    qnt=($2)14fi15 16if [ ! -z "$3" ]; then17    args="$3"18fi19 20model="$1"21out="../tmp/results-${model}"22 23set -o pipefail24set -e25 26mkdir -p ${out}27 28mstr=""29 30for q in ${qnt[@]}; do31    mstr="${mstr} -m ../models/${model}/ggml-model-${q}.gguf"32done33 34./bin/llama-bench ${mstr} ${args} 2> /dev/null35