buildrestart/Gitallama

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel

base_model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
adapter_id    = "buildrestart/Gitallama"

tokenizer = AutoTokenizer.from_pretrained(base_model_id)
model = AutoModelForCausalLM.from_pretrained(
    base_model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    load_in_4bit=True,
)
model = PeftModel.from_pretrained(model, adapter_id)
model.eval()

SYSTEM_PROMPT = (
    "You are a knowledgeable guide on the Bhagavad Gita, the ancient Hindu scripture "
    "containing the dialogue between Prince Arjuna and Lord Krishna. Provide thoughtful, "
    "accurate explanations of its verses, teachings, and philosophical concepts."
)

messages = [
    {"role": "system",  "content": SYSTEM_PROMPT},
    {"role": "user",    "content": "What does the Bhagavad Gita teach about karma?"},
]

prompt = tokenizer.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

with torch.no_grad():
    output = model.generate(
        **inputs,
        max_new_tokens=500,
        temperature=0.7,
        top_p=0.9,
        do_sample=True,
        pad_token_id=tokenizer.eos_token_id,
    )

response = tokenizer.decode(
    output[0][inputs["input_ids"].shape[1]:],
    skip_special_tokens=True,
)
print(response)

import torch
from unsloth import FastLanguageModel
from peft import PeftModel

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/meta-llama-3.1-8b-instruct-bnb-4bit",
    max_seq_length=2048,
    dtype=torch.bfloat16,
    load_in_4bit=True,
)
model = PeftModel.from_pretrained(model, "buildrestart/Gitallama")
model = model.merge_and_unload()
FastLanguageModel.for_inference(model)