maxtsui/annotate

1# from src import SalesSimulator, ConversationConfig2 3# # Make conversation config4# config = ConversationConfig.make()5 6# # Simulate & Store conversation: will use default conversation config if not provided7# simsales = SalesSimulator.make(config)8# conversation_history = simsales.simulate(n=10, print_conversation=True)9 10# print('Conversation history: ', simsales.conversation_history)11 12# # Store conversation & parse it into json file13# simsales.store_conversation()14 15 16# test on pairmatch17from src import pairmatch_baseline18from data.rude_prompt import RUDE_TIPS, TEST_TIPS19# collect conversations20import glob21conversation_data_folder = './data/conversation/'22conversation_files = glob.glob(f'{conversation_data_folder}conversation_*.json')23 24# read conversations from json files25import json26conversations = []27for conversation_file in conversation_files:28    with open(conversation_file, 'r') as f:29        conversation = json.load(f)30        conversations.append(conversation)31 32# POE33i, j = 0, 134conversation_A = conversations[i]35conversation_B = conversations[j]36conversation_history_pair = (conversation_A, conversation_B)37# sub_objectives = RUDE_TIPS[9:10] # qualitative evaluation - True or Falseß38# sub_objectives = RUDE_TIPS[9:10] # quantitative evaluation39 40sub_objectives = TEST_TIPS41 42# Pairmatch with permutation & backward evaluation43# judge, info = pairmatch_permuted_backward_reflect(conversation_history_pair, sub_objectives)44# print('POE information: \n', info)45 46# Global Comparison -- global scoring with relative rankings47from src import stochastic_bubble_sort48stochastic_bubble_sort(conversations[:2], sub_objectives, 49                       store_path = f'{conversation_data_folder}sort/',50                       name = 'scores_poe_test')51 52 53 54 55 56 57 58# Forms of Bias check over multiple LLMs should be skeched out here.59# -- Confirmation Bias & First Impression Bias60# llms = ['GPT3.5', 'GPT4', 'Gemini', 'Mixtral', 'Mistral']61# llm = load_llms()62# test_confirmation_bias(conversation_history_pair, sub_objectives, llm)63# test_first_impression_bias(conversation_history_pair, sub_objectives, llm)64 65