KBaba7/llama.cpp

1#!/bin/bash2 3set -e4 5cd "$(dirname "$0")/.." || exit6 7MODEL="${MODEL:-./models/13B/ggml-model-q4_0.bin}"8PROMPT_TEMPLATE=${PROMPT_TEMPLATE:-./prompts/chat.txt}9USER_NAME="${USER_NAME:-USER}"10AI_NAME="${AI_NAME:-ChatLLaMa}"11 12# Adjust to the number of CPU cores you want to use.13N_THREAD="${N_THREAD:-8}"14# Number of tokens to predict (made it larger than default because we want a long interaction)15N_PREDICTS="${N_PREDICTS:-2048}"16 17# Note: you can also override the generation options by specifying them on the command line:18# For example, override the context size by doing: ./chatLLaMa --ctx_size 102419GEN_OPTIONS="${GEN_OPTIONS:---ctx_size 2048 --temp 0.7 --top_k 40 --top_p 0.5 --repeat_last_n 256 --batch_size 1024 --repeat_penalty 1.17647}"20 21DATE_TIME=$(date +%H:%M)22DATE_YEAR=$(date +%Y)23 24PROMPT_FILE=$(mktemp -t llamacpp_prompt.XXXXXXX.txt)25 26sed -e "s/\[\[USER_NAME\]\]/$USER_NAME/g" \27    -e "s/\[\[AI_NAME\]\]/$AI_NAME/g" \28    -e "s/\[\[DATE_TIME\]\]/$DATE_TIME/g" \29    -e "s/\[\[DATE_YEAR\]\]/$DATE_YEAR/g" \30     $PROMPT_TEMPLATE > $PROMPT_FILE31 32# shellcheck disable=SC2086 # Intended splitting of GEN_OPTIONS33./llama-cli $GEN_OPTIONS \34  --model "$MODEL" \35  --threads "$N_THREAD" \36  --n_predict "$N_PREDICTS" \37  --color --interactive \38  --file ${PROMPT_FILE} \39  --reverse-prompt "${USER_NAME}:" \40  --in-prefix ' ' \41  "$@"42