CONDA-Workshop/Data-Contamination-Database

1def load_file(filename):2    with open(filename, 'r') as f:3        header = f.readline().strip().split(";")4        return header, [line.strip().split(";") for line in f if line.strip()]5 6def remove_duplicates(data):7    keys = set()8    _data = []9    for item in data:10        key = tuple((item[0], item[1], item[2], item[3], item[-1]))11        if key in keys:12            continue13        _data += [item]14        keys.add(key)15    return _data16 17def fix_arxiv_links(data):18    return [[*item[:-2], item[-2].replace("arxiv.org/pdf", "arxiv.org/abs"), item[-1]] for item in data]19 20def fix_openreview_links(data):21    return [[*item[:-2], item[-2].replace("openreview.net/pdf", "openreview.net/forum"), item[-1]] for item in data]22 23def sort_data(data):24    return sorted(data, key=lambda x: (x[0], x[1], x[2], x[3], x[-1]))25 26def main():27    header, data = load_file("contamination_report.csv")28    data = sort_data(data)29    data = remove_duplicates(data)30    data = fix_arxiv_links(data)31    data = fix_openreview_links(data)32    print("Total datapoints:", len(data))33 34    with open("contamination_report.csv", 'w') as f:35        f.write(";".join(header) + "\n")36        past_key = None37        for line in data:38            key = tuple((line[0], line[1]))39            if key != past_key:40                f.write("\n")41                past_key = key42            line = line[:3] + line[3:]43            f.write(";".join(line) + "\n")44 45 46if __name__ == "__main__":47    main()