bigcode/commitpackmeta

1import json2import random3import os4import datasets5 6opt_out_github_login_the_stack_12 = '''tomas7bermannoah8rtzoeller9sdatkinson10jtth11benui-dev12DrTimothyAldenDavis13pranftw14marcsances15sitmo16jucor17digantamisra9818przemb19jisakiel20kellycochran21AkashM39822MeSayan23SWinxy24zitterbewegung25fredster3326demangejeremy27PPPI28henilp10529tobymcclean30cynddl31spate14132plaaosert33woylie34alexkoster35satyrnidae36rclark37zakkolar38jimscard39marjoleinF40ghchinoy41jmfadeley42fuhrmanator43stephen-three44retro-ai45abolouri46matthewheath47jollypolly12348RJRauch49ivansche50'''.splitlines()51 52paths = '''diffs_10485761_11010048.jsonl53diffs_1048577_1572864.jsonl54diffs_11010049_11534336.jsonl55diffs_11534337_12058624.jsonl56diffs_12058625_12582912.jsonl57diffs_12582913_13107200.jsonl58diffs_13107201_13631488.jsonl59diffs_13631489_14155776.jsonl60diffs_14155777_14680064.jsonl61diffs_14680065_15204352.jsonl62diffs_15204353_15728640.jsonl63diffs_15728641_16252928.jsonl64diffs_1572865_2097152.jsonl65diffs_16252929_16777216.jsonl66diffs_1_524288.jsonl67diffs_2097153_2621440.jsonl68diffs_25165825_25690112.jsonl69diffs_25690113_26214400.jsonl70diffs_26214401_26738688.jsonl71diffs_2621441_3145728.jsonl72diffs_26738689_27262976.jsonl73diffs_27262977_27787264.jsonl74diffs_27787265_28311552.jsonl75diffs_28311553_28835840.jsonl76diffs_28835841_29360128.jsonl77diffs_29360129_29884416.jsonl78diffs_29884417_30408704.jsonl79diffs_30408705_30932992.jsonl80diffs_30932993_31457280.jsonl81diffs_31457281_31981568.jsonl82diffs_3145729_3670016.jsonl83diffs_31981569_32505856.jsonl84diffs_32505857_33030144.jsonl85diffs_33030145_33554432.jsonl86diffs_33554433_34078720.jsonl87diffs_34078721_34603008.jsonl88diffs_34603009_35127296.jsonl89diffs_35127297_35651584.jsonl90diffs_35651585_36175872.jsonl91diffs_36175873_36700160.jsonl92diffs_36700161_37224448.jsonl93diffs_3670017_4194304.jsonl94diffs_37224449_37748736.jsonl95diffs_37748737_38273024.jsonl96diffs_38273025_38797312.jsonl97diffs_38797313_39321600.jsonl98diffs_39321601_39845888.jsonl99diffs_39845889_40370176.jsonl100diffs_40370177_40894464.jsonl101diffs_40894465_41418752.jsonl102diffs_41418753_41943040.jsonl103diffs_41943041_42467328.jsonl104diffs_4194305_4718592.jsonl105diffs_42467329_42991616.jsonl106diffs_42991617_43515904.jsonl107diffs_43515905_44040192.jsonl108diffs_44040193_44564480.jsonl109diffs_44564481_45088768.jsonl110diffs_45088769_45613056.jsonl111diffs_45613057_46137344.jsonl112diffs_46137345_46661632.jsonl113diffs_46661633_47185920.jsonl114diffs_47185921_47710208.jsonl115diffs_4718593_5242880.jsonl116diffs_47710209_48234496.jsonl117diffs_48234497_48758784.jsonl118diffs_48758785_49283072.jsonl119diffs_49283073_49807360.jsonl120diffs_49807361_50331648.jsonl121diffs_50331649_50855936.jsonl122diffs_50855937_51380224.jsonl123diffs_51380225_51904512.jsonl124diffs_51904513_52428800.jsonl125diffs_52428801_52953088.jsonl126diffs_5242881_5767168.jsonl127diffs_524289_1048576.jsonl128diffs_52953089_53477376.jsonl129diffs_53477377_54001664.jsonl130diffs_54001665_54525952.jsonl131diffs_54525953_55050240.jsonl132diffs_55050241_55574528.jsonl133diffs_55574529_56098816.jsonl134diffs_56098817_56623104.jsonl135diffs_56623105_57147392.jsonl136diffs_57147393_57671680.jsonl137diffs_57671681_58195968.jsonl138diffs_5767169_6291456.jsonl139diffs_58195969_58720256.jsonl140diffs_49807361_50331648.jsonl141diffs_50331649_50855936.jsonl142diffs_50855937_51380224.jsonl143diffs_51380225_51904512.jsonl144diffs_51904513_52428800.jsonl145diffs_52428801_52953088.jsonl146diffs_5242881_5767168.jsonl147diffs_524289_1048576.jsonl148diffs_52953089_53477376.jsonl149diffs_53477377_54001664.jsonl150diffs_54001665_54525952.jsonl151diffs_54525953_55050240.jsonl152diffs_55050241_55574528.jsonl153diffs_55574529_56098816.jsonl154diffs_56098817_56623104.jsonl155diffs_56623105_57147392.jsonl156diffs_57147393_57671680.jsonl157diffs_57671681_58195968.jsonl158diffs_5767169_6291456.jsonl159diffs_58195969_58720256.jsonl160diffs_58720257_59244544.jsonl161diffs_59244545_59768832.jsonl162diffs_59768833_60293120.jsonl163diffs_60293121_60817408.jsonl164diffs_60817409_61341696.jsonl165diffs_61341697_61865984.jsonl166diffs_61865985_62390272.jsonl167diffs_62390273_62914560.jsonl168diffs_62914561_63438848.jsonl169diffs_6291457_6815744.jsonl170diffs_63438849_63963136.jsonl171diffs_63963137_64487424.jsonl172diffs_64487425_65011712.jsonl173diffs_65011713_65536000.jsonl174diffs_65536001_66060288.jsonl175diffs_66060289_66584576.jsonl176diffs_66584577_67108864.jsonl177diffs_6815745_7340032.jsonl178diffs_7340033_7864320.jsonl179diffs_7864321_8388608.jsonl180diffs_8388609_8912896.jsonl181diffs_8912897_9437184.jsonl182diffs_9437185_9961472.jsonl'''.splitlines()183 184 185if False:#__name__ == "__main__":186    data = {}187    # licenses:188    # {"repos":"samoht/ocaml-conduit,samoht/ocaml-conduit,avsm/ocaml-conduit,mirage/ocaml-conduit,djs55/ocaml-conduit,rgrinberg/ocaml-conduit,rgrinberg/ocaml-conduit,djs55/ocaml-conduit","license":"isc"}189    with open('licenses_merged.jsonl', 'r') as f:190        for line in f:191            line = line.strip()192            if not line:193                continue194            d = json.loads(line)195            for repo in d['repos'].split(","):196                data[repo] = d['license']197 198    NUM_PROC = 8199 200    # path = "diffs_9961473_10485760.jsonl"201    for path in paths:202        ds = datasets.load_dataset("json", data_files=path)203 204        print(ds)205        print(random.sample(data.items(), 10))206 207        def map_col(example):208            for repo in example["repos"].split(","):209                if repo in data:210                    example["license"] = data[repo]211                    return example212            example["license"] = 'unknown'213            return example214 215        def opt_out(example):216            repos = example["repos"].split(",")217            if (len(repos) == 1) and (repos[0].split("/")[0] in opt_out_github_login_the_stack_12):218                return False219            return True220        221        ds['train'].map(map_col).filter(opt_out).to_json(path.replace(".jsonl", "_new.jsonl"), num_proc=NUM_PROC)222 223"""224  AND ( (lang_table.language_name LIKE 'Python')225    OR (lang_table.language_name LIKE 'Java')226    OR (lang_table.language_name LIKE 'JavaScript')227    OR (lang_table.language_name LIKE 'HTML')228    OR (lang_table.language_name LIKE 'Common Lisp')229    OR (lang_table.language_name LIKE 'Shell')230    OR (lang_table.language_name LIKE 'R')231    OR (lang_table.language_name LIKE 'Perl%')232    OR (lang_table.language_name LIKE 'SQL')233    OR (lang_table.language_name LIKE 'C')234    OR (lang_table.language_name LIKE 'C#')235    OR (lang_table.language_name LIKE 'C++')236    OR (lang_table.language_name LIKE 'TypeScript')237    OR (lang_table.language_name LIKE 'Go')238    OR (lang_table.language_name LIKE 'Rust')239    OR (lang_table.language_name LIKE 'Swift')240    OR (lang_table.language_name LIKE 'PHP')241    OR (lang_table.language_name LIKE 'Dart')242    OR (lang_table.language_name LIKE 'Kotlin')243    OR (lang_table.language_name LIKE 'Matlab')244    OR (lang_table.language_name LIKE 'MATLAB')245    OR (lang_table.language_name LIKE 'Ruby') )246"""247 248if __name__ == "__main__":249 250    with open("programming_languages.json", "r") as f:251        prog_langs = json.load(f)252    253    ext_to_lang = {}254    for lang, exts in prog_langs.items():255        for ext in exts:256            ext_to_lang[ext] = lang257    258 259    LANGS = [260        'Python',261        'Java',262        'JavaScript',263        'HTML',264        'Common Lisp',265        'Shell',266        'R',267        'Perl',268        'Perl6', # Differentiate Perl6 from Perl269        'SQL',270        'C',271        'C#',272        'C++',273        'TypeScript',274        'Go',275        'Rust',276        'Swift',277        'PHP',278        'Dart',279        'Kotlin',280        'Matlab',281        'MATLAB',282        'Ruby'283    ]284 285    for lang in LANGS:286        if lang not in prog_langs:287            print("Missing", lang)288 289    def add_lang(example):290        example["language"] = ext_to_lang.get(example["new_file"].split(".")[-1], "unknown")291        return example292 293    for path in paths:294        ds = datasets.load_dataset("json", data_files=path)295        ds = ds['train'].map(add_lang)296        vals = ds.unique('language')297        for val in vals:298            ds.filter(lambda x: x['language'] == val).to_json(path.replace(".jsonl", f"_{val}.jsonl"), num_proc=NUM_PROC)299 300    301    302    303