luulinh90s/Tabular-LLM-Study-Preference-Ranking

1import os2import shutil3from pathlib import Path4import re5from bs4 import BeautifulSoup6 7 8def process_html_file(file_path, output_path):9    with open(file_path, 'r', encoding='utf-8') as file:10        content = file.read()11 12    # Remove lines containing 'Prediction'13    content = re.sub(r'<h3>Prediction:.*?</h3>\n?', '', content, flags=re.DOTALL)14 15    # Parse the HTML content16    soup = BeautifulSoup(content, 'html.parser')17 18    # Remove tables with 'verification_result' column19    for table in soup.find_all('table'):20        if table.find('td', string='verification_result'):21            table.decompose()22 23    # Write the modified content24    with open(output_path, 'w', encoding='utf-8') as file:25        file.write(str(soup))26 27 28def process_directory(input_dir, output_dir):29    for root, dirs, files in os.walk(input_dir):30        for file in files:31            if file.endswith('.html'):32                input_path = Path(root) / file33                relative_path = input_path.relative_to(input_dir)34                output_path = Path(output_dir) / relative_path35 36                output_path.parent.mkdir(parents=True, exist_ok=True)37                process_html_file(input_path, output_path)38 39 40# Define input and output directories41input_directory = "htmls_POS"42output_directory = "htmls_POS_mod2"43 44# Process the files45process_directory(input_directory, output_directory)46 47print("Processing complete for POS. Modified files are in the output directory.")