zihaowei/Human_Study_for_Audio_Visual_Matching

1import gradio as gr2import json3import os4import random5import time6from datetime import datetime7from huggingface_hub import Repository8import subprocess9 10# Set Git user information11subprocess.run(["git", "config", "--global", "user.email", "zihaowei@umich.edu"])12subprocess.run(["git", "config", "--global", "user.name", "Zi-hao-Wei"])13 14hf_token = os.getenv("HF_TOKEN")15print("HF Token is none?", hf_token is None)16 17# Define questions for both sets18audio_image_questions = ["Ignore the image quality, which image best matches the audio?"]19image_audio_questions = ["Ignore the audio quality, which audio best matches the image?"]20 21# Load audio-image pairs and image-audio pairs with random sampling22def load_random_pairs():23    random.seed(time.time())24 25    audio_image_pairs = []26    audio_image_mappings = {}27    for name in random.sample(os.listdir("./pick_visual/audio_real/"), k=10):28        fn = name[:-5]29        print(name)30        pair = (31            f"./pick_visual/audio_real/{name}",32            f"./pick_visual/frame_real/{fn}.jpg",33            f"./pick_visual/frame_syn/{fn}.jpg"34        )35        mapping = ("Image 1", "Image 2")36        if random.random() > 0.5:37            pair = (pair[0], pair[2], pair[1])  # Swap image paths38            mapping = ("Image 2", "Image 1")  # Adjust mapping39        audio_image_pairs.append(pair)40        audio_image_mappings[pair[0]] = {}41        audio_image_mappings[pair[0]]["Image 1"] = mapping[0]42        audio_image_mappings[pair[0]]["Image 2"] = mapping[1]43        audio_image_mappings[pair[0]]["Both good"] = "Both good"44        audio_image_mappings[pair[0]]["Both bad"] = "Both bad"45 46    image_audio_pairs = []47    image_audio_mappings = {}48    for name in random.sample(os.listdir("./pick_audio/frame_real/"), k=10):49        fn = name[:-4]50        pair = (51            f"./pick_audio/frame_real/{name}",52            f"./pick_audio/audio_real/{fn}.flac",53            f"./pick_audio/audio_syn/{fn}.flac"54        )55        mapping = ("Audio 1", "Audio 2")56        if random.random() > 0.5:57            pair = (pair[0], pair[2], pair[1])  # Swap audio paths58            mapping = ("Audio 2", "Audio 1")  # Adjust mapping59        image_audio_pairs.append(pair)60        image_audio_mappings[pair[0]] = {}61        image_audio_mappings[pair[0]]["Audio 1"] = mapping[0]62        image_audio_mappings[pair[0]]["Audio 2"] = mapping[1]63        image_audio_mappings[pair[0]]["Both good"] = "Both good"64        image_audio_mappings[pair[0]]["Both bad"] = "Both bad"65 66    return audio_image_pairs, audio_image_mappings, image_audio_pairs, image_audio_mappings67 68# Initialize the Hugging Face repository69repo_url = "https://huggingface.co/datasets/zihaowei/user_response"70repo = Repository(local_dir="user_responses", clone_from=repo_url, use_auth_token=hf_token)71 72def save_responses(*responses):73    global audio_image_mappings74    global audio_image_pairs75    global image_audio_mappings76    global image_audio_pairs77    session_id = f"session_{int(time.time())}"78    data = {79        "user_id": session_id,80        "timestamp": datetime.now().isoformat(),81        "audio_image_responses": [],82        "image_audio_responses": []83    }84 85    # Split responses for audio-image and image-audio pairs86    num_audio_image_responses = len(audio_image_pairs)87    audio_image_responses = responses[:num_audio_image_responses]88    image_audio_responses = responses[num_audio_image_responses:]89 90    # Organize responses for each pair91    for i, (audio_path, image1_path, image2_path) in enumerate(audio_image_pairs):92        selected_image = audio_image_responses[i]93        original_image_index = audio_image_mappings[audio_path][selected_image]94        data["audio_image_responses"].append({95            "audio_file": os.path.basename(audio_path),96            "selected_image": f"{original_image_index}"97        })98    99    for i, (image_path, audio1_path, audio2_path) in enumerate(image_audio_pairs):100        selected_audio = image_audio_responses[i]101        original_audio_index = image_audio_mappings[image_path][selected_audio]102        data["image_audio_responses"].append({103            "image_file": os.path.basename(image_path),104            "selected_audio": f"{original_audio_index}"105        })106 107    response_file = f"user_responses/{session_id}_responses.json"108    os.makedirs("user_responses", exist_ok=True)109    with open(response_file, "w") as f:110        json.dump(data, f, indent=4)111 112    repo.push_to_hub()113 114    return "All responses saved! Thank you."115 116def load_and_update():117    global audio_image_mappings118    global audio_image_pairs119    global image_audio_mappings120    global image_audio_pairs121    audio_image_pairs, audio_image_mappings, image_audio_pairs, image_audio_mappings = load_random_pairs()122    123    flat_output = []124    for audio, image1, image2 in audio_image_pairs:125        flat_output.extend([audio, image1, image2])126    for image, audio1, audio2 in image_audio_pairs:127        flat_output.extend([image, audio1, audio2])128    return flat_output129 130def create_interface():131    with gr.Blocks() as demo:132        gr.Markdown("## Audio-Image Matching")133        gr.Markdown("Listen to each audio sample and select the image that best matches it.")134        audio_image_components = []135        audio_image_responses = []136        for idx in range(10):137            gr.Markdown(f"### Pair {idx + 1}")138            with gr.Row():139                audio = gr.Audio(label="Reference audio")140                image1 = gr.Image(label="Image 1")141                image2 = gr.Image(label="Image 2")142                radio = gr.Radio(["Image 1", "Image 2", "Both good", "Both bad"], label=audio_image_questions[0])143            gr.Markdown("---")144            audio_image_components.extend([audio, image1, image2])145            audio_image_responses.append(radio)146 147        gr.Markdown("## Image-Audio Matching")148        gr.Markdown("View each image and select the audio sample that best matches it.")149        image_audio_components = []150        image_audio_responses = []151        for idx in range(10):152            gr.Markdown(f"### Pair {idx + 1}")153            with gr.Row():154                image = gr.Image(label="Reference image")155                audio1 = gr.Audio(label="Audio 1")156                audio2 = gr.Audio(label="Audio 2")157                radio = gr.Radio(["Audio 1", "Audio 2", "Both good", "Both bad"], label=image_audio_questions[0])158            gr.Markdown("---")159            image_audio_components.extend([image, audio1, audio2])160            image_audio_responses.append(radio)161 162        submit_btn = gr.Button("Submit All")163        result_message = gr.Textbox(label="Message", interactive=False)164        165        def validate_and_save(*responses):166            if any(response is None for response in responses):167                return "Please answer all questions before submitting."168            return save_responses(*responses)169 170        demo.load(171            fn=load_and_update,172            inputs=None,173            outputs=audio_image_components[:30] + image_audio_components[:30]174        )175 176        submit_btn.click(177            fn=validate_and_save,178            inputs=audio_image_responses + image_audio_responses,179            outputs=result_message180        )181 182    return demo183 184demo = create_interface()185demo.launch(share=True)