bigcode/commitpackmeta

1import random2import subprocess3 4import datasets5 6"""7git init8git remote add origin https://github.com/huggingface/evaluate.git9git fetch --depth 2 origin 9b056cdd5eb95459ae80142014865263e7dd75b810# Get file after change11git checkout FETCH_HEAD -- README.md12# Get file before change13git checkout FETCH_HEAD^ -- README.md14"""15 16#Shell utils17def run_in_shell(cmd: str, cwd=None, timeout=10):18    # Default 10sec timeout (mostly for git fetch --depth 2 origin when the repo is private & GH asks for a username)19    completed = subprocess.run([cmd], capture_output=True, shell=True, cwd=cwd, timeout=timeout)20    return completed21 22def get_file_contents(commit, old_file, new_file, repo, cwd=None):23 24    completed = run_in_shell("git init", cwd=cwd)25    completed = run_in_shell("git remote add origin " + repo, cwd=cwd)26    completed = run_in_shell("git fetch --depth 2 origin " + commit, cwd=cwd)27    # If it times out as repo requires a username28    if completed.returncode != 0:29        return ""30    git_diff = run_in_shell(f"git diff {commit}^ {commit}", cwd=cwd).stdout.decode()31    return git_diff32 33def get_diff(ex):34    commit_id = ex["commit"]35    repos = list(set(ex["repos"].split(",")))36    old_file = ex["old_file"]37    new_file = ex["new_file"]38    for repo in repos:39        repo = "https://www.github.com/" + repo + ".git"40        # create a random directory to store the repo41        random_dir = random.randint(0, 1000000)42        run_in_shell("mkdir " + str(random_dir))43        try:44            git_diff = get_file_contents(commit_id, old_file, new_file, repo, cwd=str(random_dir))45        except Exception as e:46            print("ERROR", commit_id, old_file, new_file, repo, str(random_dir), e)47            run_in_shell("rm -rf " + str(random_dir))48            continue49        ex["diff"] = git_diff50        run_in_shell("rm -rf " + str(random_dir))51        return ex52    # If no repo worked53    ex["diff"] = ""54    return ex55 56if __name__ == "__main__":57    ds = datasets.load_dataset("bigcode/github-commits", use_auth_token=True).shuffle()58 59    diff_ds = ds["train"].select(range(128)).map(get_diff, num_proc=128)60    # diff_ds.push_to_hub("bigcode/commits", use_auth_token=True) # Not working61    diff_ds.to_json("gitdiffs.jsonl")