FosterSystemsDatabase/ReinforcementLearning

1from unsloth import FastLanguageModel2from peft import PeftModel3import pandas as pd4from unsloth.chat_templates import get_chat_template5from sklearn.feature_extraction.text import TfidfVectorizer6from sklearn.metrics.pairwise import cosine_similarity7from sentence_transformers import SentenceTransformer, util8import nltk9import json10from google.oauth2.service_account import Credentials11import gspread12import gradio as gr13 14# Download stopwords15nltk.download("stopwords")16from nltk.corpus import stopwords17 18# Load the base model with FastLanguageModel19model, tokenizer = FastLanguageModel.from_pretrained(20    model_name="unsloth/Llama-3.2-3B-Instruct",21    max_seq_length=2048,22    dtype=None,23    load_in_4bit=True24)25adapter_path = "FosterSystemsDatabase/model"26model = PeftModel.from_pretrained(model, adapter_path)27 28# Load CSV data29file_path = 'Clean Missouri Data.csv'30df = pd.read_csv(file_path, encoding='MacRoman')31 32def search_relevant_policies(query, df, top_n=10, max_chars=40000):33    tfidf = TfidfVectorizer(stop_words='english')34    tfidf_matrix = tfidf.fit_transform(df['Content'])35    query_vector = tfidf.transform([query])36    cosine_sim = cosine_similarity(query_vector, tfidf_matrix).flatten()37 38    top_indices = cosine_sim.argsort()[-top_n:][::-1]39    relevant_policies = df.iloc[top_indices].copy()40 41    char_count = 042    valid_indices = []43    for idx, row in relevant_policies.iterrows():44        content_length = len(row["Content"])45        if char_count + content_length > max_chars:46            break47        char_count += content_length48        valid_indices.append(idx)49 50    truncated_policies = relevant_policies.loc[valid_indices]51    return truncated_policies52 53def get_content_after_query(response_text, query):54    query_position = response_text.lower().find(query.lower())55    if query_position != -1:56        res = response_text[query_position + len(query):].strip()57        return res[11:]58    else:59        return response_text.strip()60 61def process_query(query, tokenizer):62    relevant_policies = search_relevant_policies(query, df)63    formatted_policies = [row['Content'] for _, row in relevant_policies.iterrows()]64    relevant_policy_text = "\n\n".join(formatted_policies)65 66    messages_with_relevant_policies = [67        {"role": "system", "content": relevant_policy_text},68        {"role": "user", "content": query},69    ]70 71    tokenizer = get_chat_template(tokenizer, chat_template="llama-3.1")72    inputs = tokenizer.apply_chat_template(73        messages_with_relevant_policies,74        tokenize=True,75        add_generation_prompt=True,76        return_tensors="pt"77    ).to("cuda")78 79    FastLanguageModel.for_inference(model)80    outputs = model.generate(input_ids=inputs, max_new_tokens=512, use_cache=True, temperature=0.7, min_p=0.1)81    generated_response = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]82    response = get_content_after_query(generated_response, query)83 84    model_sbert = SentenceTransformer('all-MiniLM-L6-v2')85    response_embedding = model_sbert.encode(generated_response, convert_to_tensor=True)86    policy_embeddings = model_sbert.encode(relevant_policies['Content'].tolist(), convert_to_tensor=True)87    cosine_similarities = util.cos_sim(response_embedding, policy_embeddings).flatten()88    most_relevant_index = cosine_similarities.argmax().item()89    most_relevant_link = relevant_policies.iloc[most_relevant_index]['Link to Content']90 91    return {92        "response": response,93        "most_relevant_link": most_relevant_link94    }95 96# Set up Google Sheets97json_file_path = "fostercare-449201-85282f81c3b7.json"98with open(json_file_path, 'r') as file:99    service_account_data = json.load(file)100scopes = ["https://www.googleapis.com/auth/spreadsheets", "https://www.googleapis.com/auth/drive"]101creds = Credentials.from_service_account_info(service_account_data, scopes=scopes)102client = gspread.authorize(creds)103spreadsheet = client.open("Fostercare Responses").sheet1104 105# Gradio functions106def greet(query):107    result_1 = process_query(query, tokenizer)108    result_2 = process_query(query, tokenizer)109    return [result_1["response"], result_2["response"]]110 111def choose_preference(name, output1, output2, preference, query, broken):112    if not name:113        return "Please enter your name before submitting."114 115    broken_flag = "Yes" if broken else "No"116 117    if preference == "Output 1":118        new_row = [query, output1, output2, name, broken_flag]119        spreadsheet.append_row(new_row)120        return f"You preferred: Output 1 - {output1}"121    elif preference == "Output 2":122        new_row = [query, output2, output1, name, broken_flag]123        spreadsheet.append_row(new_row)124        return f"You preferred: Output 2 - {output2}"125    else:126        return "No preference selected."127 128# Gradio UI129with gr.Blocks() as demo:130    name_input = gr.Textbox(label="Enter your name")131    query_input = gr.Textbox(label="Enter your query")132    generate_button = gr.Button("Generate Outputs")133    output_1 = gr.Textbox(label="Output 1", interactive=False)134    output_2 = gr.Textbox(label="Output 2", interactive=False)135    preference = gr.Radio(["Output 1", "Output 2"], label="Choose your preferred output")136    broken_flag = gr.Checkbox(label="Mark as Broken Response")137    preference_result = gr.Textbox(label="Preference Result", interactive=False)138    submit_button = gr.Button("Submit Preference")139 140    generate_button.click(greet, inputs=query_input, outputs=[output_1, output_2])141    submit_button.click(142        choose_preference,143        inputs=[name_input, output_1, output_2, preference, query_input, broken_flag],144        outputs=preference_result145    ).then(146        fn=lambda: ("", "", "", "", "", False, ""),147        inputs=[],148        outputs=[name_input, query_input, output_1, output_2, preference, broken_flag, preference_result]149    )150 151demo.launch()