K00B404/Merged_Models_Test_Chat

1#from huggingface_hub import InferenceClient2import gradio as gr3 4 5 6#client = InferenceClient("""K00B404/BagOMistral_14X_Coders-ties-7B""")7from transformers import AutoTokenizer, AutoModelForSequenceClassification8import torch9from transformers import AutoModelForCausalLM, AutoTokenizer10 11#model_id = 'EleutherAI/pythia-1b' # 1612model_id = 'EleutherAI/GPT-Neo-2.7B' #32 layers13#gr.load(f"models/{model_id}").launch()14tokenizer = AutoTokenizer.from_pretrained(model_id)15 16model = AutoModelForCausalLM.from_pretrained(model_id)17 18text = "TinyPixel/Llama-2-7B-bf16-sharded"19inputs = tokenizer(text, return_tensors="pt")20 21outputs = model.generate(**inputs, max_new_tokens=20)22print(tokenizer.decode(outputs[0], skip_special_tokens=True))23 24 25"""26def format_prompt(message, history):27  prompt = "<s>"28  for user_prompt, bot_response in history:29    prompt += f"[INST] {user_prompt} [/INST]"30    prompt += f" {bot_response}</s> "31    prompt += f"[INST] {message} [/INST]"32  return prompt33 34def generate(prompt, history, temperature=0.2, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0):35    temperature = float(temperature)36    if temperature < 1e-2:37        temperature = 1e-238    top_p = float(top_p)39 40    generate_kwargs = dict(41        temperature=temperature,42        max_new_tokens=max_new_tokens,43        top_p=top_p,44        repetition_penalty=repetition_penalty,45        do_sample=True,46        seed=42,47    )48 49    formatted_prompt = format_prompt(prompt, history)50 51    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)52    output = ""53 54    for response in stream:55        output += response.token.text56        yield output57    return output58 59    60mychatbot = gr.Chatbot(avatar_images=["./user.png", "./botm.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True)61 62demo = gr.ChatInterface(fn=generate, 63                        chatbot=mychatbot,64                        title="K00B404's Merged Models Test Chat",65                        retry_btn=None,66                        undo_btn=None67                       )68 69demo.queue().launch(show_api=False)70"""