bigcode/commitpackmeta

1import random2import subprocess3 4import datasets5 6"""7git init8git remote add origin https://github.com/huggingface/evaluate.git9git fetch --depth 2 origin 9b056cdd5eb95459ae80142014865263e7dd75b810# Get file after change11git checkout FETCH_HEAD -- README.md12# Get file before change13git checkout FETCH_HEAD^ -- README.md14"""15 16#Shell utils17def run_in_shell(cmd: str, cwd=None, timeout=120):18    # Default 2min timeout (mostly for git fetch --depth 2 origin when the repo is private & GH asks for a username)19    completed = subprocess.run([cmd], capture_output=True, shell=True, cwd=cwd, timeout=timeout)20    return completed21 22def get_file_contents(commit, old_file, new_file, repo, cwd=None):23 24    completed = run_in_shell("git init", cwd=cwd)25    completed = run_in_shell("git remote add origin " + repo, cwd=cwd)26    completed = run_in_shell("git fetch --depth 2 origin " + commit, cwd=cwd)27    # If it times out as repo requires a username28    if completed.returncode != 0:29        return (new_contents, "")30    old_contents = run_in_shell("cat " + old_file, cwd=cwd).stdout.decode()31    completed = run_in_shell("git checkout FETCH_HEAD -- " + new_file, cwd=cwd)32    new_contents = run_in_shell("cat " + new_file, cwd=cwd).stdout.decode()33    completed = run_in_shell("git checkout FETCH_HEAD^ -- " + old_file, cwd=cwd)34    # If there's only a new file, but no old file35    if completed.returncode != 0:36        return (new_contents, "")37    old_contents = run_in_shell("cat " + old_file, cwd=cwd).stdout.decode()38    return (new_contents, old_contents)39 40def get_diff(ex):41    commit_id = ex["commit"]42    repos = list(set(ex["repos"].split(",")))43    old_file = ex["old_file"]44    new_file = ex["new_file"]45    for repo in repos:46        repo = "https://www.github.com/" + repo + ".git"47        # create a random directory to store the repo48        random_dir = random.randint(0, 1000000)49        run_in_shell("mkdir " + str(random_dir))50        try:51            new_contents, old_contents = get_file_contents(commit_id, old_file, new_file, repo, cwd=str(random_dir))52        except Exception as e:53            print("ERROR", commit_id, old_file, new_file, repo, str(random_dir), e)54            run_in_shell("rm -rf " + str(random_dir))55            continue56        ex["new_contents"] = new_contents57        ex["old_contents"] = old_contents58        run_in_shell("rm -rf " + str(random_dir))59        return ex60    # If no repo worked61    ex["new_contents"] = ""62    ex["old_contents"] = ""63    return ex64 65if __name__ == "__main__":66    ds = datasets.load_dataset("bigcode/github-commits", use_auth_token=True).shuffle()67 68    diff_ds = ds["train"].map(get_diff, num_proc=128)69    # diff_ds.push_to_hub("bigcode/commits", use_auth_token=True) # Not working70    diff_ds.to_json("diffs.jsonl")71