zac/Coding_with_LLAMA_CPU

1import gradio as gr2import copy3import time4import ctypes #to run on C api directly 5import llama_cpp6from llama_cpp import Llama7from huggingface_hub import hf_hub_download #load from huggingfaces 8 9 10llm = Llama(model_path= hf_hub_download(repo_id="TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML", filename="llama-2-7b-chat-codeCherryPop.ggmlv3.q6_K.bin"), n_ctx=2048) #download model from hf/ n_ctx=2048 for high ccontext length11 12history = []13 14pre_prompt = " The user and the AI are having a conversation : <|endoftext|> \n "15 16def generate_text(input_text, history):17    print("history ",history)18    print("input ", input_text)19    temp =""20    if history == []:21        input_text_with_history = f"SYSTEM:{pre_prompt}"+ "\n" + f"USER: {input_text} " + "\n" +" ASSISTANT:"22    else:23        input_text_with_history = f"{history[-1][1]}"+ "\n"24        input_text_with_history += f"USER: {input_text}" + "\n" +" ASSISTANT:"25    print("new input", input_text_with_history)26    output = llm(input_text_with_history, max_tokens=1024, stop=["<|prompter|>", "<|endoftext|>", "<|endoftext|> \n","ASSISTANT:","USER:","SYSTEM:"], stream=True)27    for out in output:28     stream = copy.deepcopy(out)29     print(stream["choices"][0]["text"])30     temp += stream["choices"][0]["text"]31     yield temp32 33 34    history =["init",input_text_with_history]35        36 37 38demo = gr.ChatInterface(generate_text,39    title="LLM on CPU",40    description="Running LLM with https://github.com/abetlen/llama-cpp-python. btw the text streaming thing was the hardest thing to impliment",41    examples=["Hello", "Am I cool?", "Are tomatoes vegetables?"],42    cache_examples=True,43    retry_btn=None,44    undo_btn="Delete Previous",45    clear_btn="Clear",)46demo.queue(concurrency_count=1, max_size=5)47demo.launch()48 49 50 51 52