OSS-forge/CodeQualityEval

1import time2import subprocess3import os4import argparse5import shutil6from pathlib import Path7 8def run_semgrep_analysis(temp_dir, json_filename):9    start_time = time.time()10 11    print(f"Running Semgrep analysis on files in {temp_dir} and saving results to {json_filename}...")12    original_dir = os.getcwd()13 14    try:15        os.chdir(temp_dir)16 17        semgrep_command = [18            "semgrep", "scan",19            "--verbose",20            "--output", json_filename,21            "--json",22            "-j", "21",23            "--no-git-ignore",24            "--max-memory=30000",25            "--max-target-bytes=1000000",26            "--timeout-threshold", "10",27            "--timeout", "60",28            "--metrics", "off",29            "--config", "p/trailofbits",30            "--config", "p/default",31            "--config", "p/comment",32            "--config", "p/java",33            "--config", "p/cwe-top-25",34            "--config", "p/owasp-top-ten",35            "--config", "p/r2c-security-audit",36            "--config", "p/insecure-transport",37            "--config", "p/secrets",38            "--config", "p/findsecbugs",39            "--config", "p/gitlab",40            "--config", "p/mobsfscan",41            "--config", "p/command-injection",42            "--config", "p/sql-injection",43            "."44        ]45        46        subprocess.run(semgrep_command, check=True)47    finally:48        os.chdir(original_dir)49 50    end_time = time.time()51    run_semgrep_time = end_time - start_time52    return run_semgrep_time53 54def batch_files(input_folder, batch_size):55    """Yields batches of files from the input folder."""56    java_files = list(Path(input_folder).rglob("*.java"))57    for i in range(0, len(java_files), batch_size):58        yield java_files[i:i + batch_size]59 60if __name__ == "__main__":61    parser = argparse.ArgumentParser(description='Batch process Java files and run Semgrep analysis.')62    parser.add_argument('dataset_name', type=str, help='The dataset name for output files.')63    parser.add_argument('batch_size', type=int, help='Number of files to process per batch.')64    args = parser.parse_args()65 66    input_folder = "./java_temp_wrapped"67    output_folder = "./semgrep_batches"  68    temp_dir = "./temp_batch" 69    dataset_name = args.dataset_name70    batch_size = args.batch_size71 72    Path(output_folder).mkdir(parents=True, exist_ok=True)73 74    for batch_index, batch in enumerate(batch_files(input_folder, batch_size)):75        if Path(temp_dir).exists():76            shutil.rmtree(temp_dir)77        Path(temp_dir).mkdir(parents=True, exist_ok=True)78 79        for file in batch:80            shutil.copy(file, temp_dir)81 82        json_filename = os.path.abspath(os.path.join(output_folder, f"{dataset_name}_semgrep_results_batch_{batch_index+1}.json"))83 84        try:85            batch_time = run_semgrep_analysis(temp_dir, json_filename)86            print(f"Batch {batch_index+1} completed in {batch_time:.2f} minutes ({batch_time/60:.2f} hrs).")87        except Exception as e:88            print(f"Error processing batch {batch_index+1}: {e}")89 90        shutil.rmtree(temp_dir)