documatic/code_t5_small_git_diff_7k_dataset
033
1{2 "version": "1.0",3 "truncation": {4 "direction": "Right",5 "max_length": 128,6 "strategy": "LongestFirst",7 "stride": 08 },9 "padding": {10 "strategy": "BatchLongest",11 "direction": "Right",12 "pad_to_multiple_of": null,13 "pad_id": 0,14 "pad_type_id": 0,15 "pad_token": "<pad>"16 },17 "added_tokens": [18 {19 "id": 0,20 "content": "<pad>",21 "single_word": false,22 "lstrip": false,23 "rstrip": false,24 "normalized": true,25 "special": true26 },27 {28 "id": 1,29 "content": "<s>",30 "single_word": false,31 "lstrip": false,32 "rstrip": false,33 "normalized": true,34 "special": true35 },36 {37 "id": 2,38 "content": "</s>",39 "single_word": false,40 "lstrip": false,41 "rstrip": false,42 "normalized": true,43 "special": true44 },45 {46 "id": 3,47 "content": "<unk>",48 "single_word": false,49 "lstrip": false,50 "rstrip": false,51 "normalized": true,52 "special": true53 },54 {55 "id": 4,56 "content": "<mask>",57 "single_word": false,58 "lstrip": true,59 "rstrip": false,60 "normalized": true,61 "special": true62 },63 {64 "id": 32000,65 "content": "<extra_id_99>",66 "single_word": false,67 "lstrip": true,68 "rstrip": false,69 "normalized": true,70 "special": true71 },72 {73 "id": 32001,74 "content": "<extra_id_98>",75 "single_word": false,76 "lstrip": true,77 "rstrip": false,78 "normalized": true,79 "special": true80 },81 {82 "id": 32002,83 "content": "<extra_id_97>",84 "single_word": false,85 "lstrip": true,86 "rstrip": false,87 "normalized": true,88 "special": true89 },90 {91 "id": 32003,92 "content": "<extra_id_96>",93 "single_word": false,94 "lstrip": true,95 "rstrip": false,96 "normalized": true,97 "special": true98 },99 {100 "id": 32004,101 "content": "<extra_id_95>",102 "single_word": false,103 "lstrip": true,104 "rstrip": false,105 "normalized": true,106 "special": true107 },108 {109 "id": 32005,110 "content": "<extra_id_94>",111 "single_word": false,112 "lstrip": true,113 "rstrip": false,114 "normalized": true,115 "special": true116 },117 {118 "id": 32006,119 "content": "<extra_id_93>",120 "single_word": false,121 "lstrip": true,122 "rstrip": false,123 "normalized": true,124 "special": true125 },126 {127 "id": 32007,128 "content": "<extra_id_92>",129 "single_word": false,130 "lstrip": true,131 "rstrip": false,132 "normalized": true,133 "special": true134 },135 {136 "id": 32008,137 "content": "<extra_id_91>",138 "single_word": false,139 "lstrip": true,140 "rstrip": false,141 "normalized": true,142 "special": true143 },144 {145 "id": 32009,146 "content": "<extra_id_90>",147 "single_word": false,148 "lstrip": true,149 "rstrip": false,150 "normalized": true,151 "special": true152 },153 {154 "id": 32010,155 "content": "<extra_id_89>",156 "single_word": false,157 "lstrip": true,158 "rstrip": false,159 "normalized": true,160 "special": true161 },162 {163 "id": 32011,164 "content": "<extra_id_88>",165 "single_word": false,166 "lstrip": true,167 "rstrip": false,168 "normalized": true,169 "special": true170 },171 {172 "id": 32012,173 "content": "<extra_id_87>",174 "single_word": false,175 "lstrip": true,176 "rstrip": false,177 "normalized": true,178 "special": true179 },180 {181 "id": 32013,182 "content": "<extra_id_86>",183 "single_word": false,184 "lstrip": true,185 "rstrip": false,186 "normalized": true,187 "special": true188 },189 {190 "id": 32014,191 "content": "<extra_id_85>",192 "single_word": false,193 "lstrip": true,194 "rstrip": false,195 "normalized": true,196 "special": true197 },198 {199 "id": 32015,200 "content": "<extra_id_84>",201 "single_word": false,202 "lstrip": true,203 "rstrip": false,204 "normalized": true,205 "special": true206 },207 {208 "id": 32016,209 "content": "<extra_id_83>",210 "single_word": false,211 "lstrip": true,212 "rstrip": false,213 "normalized": true,214 "special": true215 },216 {217 "id": 32017,218 "content": "<extra_id_82>",219 "single_word": false,220 "lstrip": true,221 "rstrip": false,222 "normalized": true,223 "special": true224 },225 {226 "id": 32018,227 "content": "<extra_id_81>",228 "single_word": false,229 "lstrip": true,230 "rstrip": false,231 "normalized": true,232 "special": true233 },234 {235 "id": 32019,236 "content": "<extra_id_80>",237 "single_word": false,238 "lstrip": true,239 "rstrip": false,240 "normalized": true,241 "special": true242 },243 {244 "id": 32020,245 "content": "<extra_id_79>",246 "single_word": false,247 "lstrip": true,248 "rstrip": false,249 "normalized": true,250 "special": true251 },252 {253 "id": 32021,254 "content": "<extra_id_78>",255 "single_word": false,256 "lstrip": true,257 "rstrip": false,258 "normalized": true,259 "special": true260 },261 {262 "id": 32022,263 "content": "<extra_id_77>",264 "single_word": false,265 "lstrip": true,266 "rstrip": false,267 "normalized": true,268 "special": true269 },270 {271 "id": 32023,272 "content": "<extra_id_76>",273 "single_word": false,274 "lstrip": true,275 "rstrip": false,276 "normalized": true,277 "special": true278 },279 {280 "id": 32024,281 "content": "<extra_id_75>",282 "single_word": false,283 "lstrip": true,284 "rstrip": false,285 "normalized": true,286 "special": true287 },288 {289 "id": 32025,290 "content": "<extra_id_74>",291 "single_word": false,292 "lstrip": true,293 "rstrip": false,294 "normalized": true,295 "special": true296 },297 {298 "id": 32026,299 "content": "<extra_id_73>",300 "single_word": false,301 "lstrip": true,302 "rstrip": false,303 "normalized": true,304 "special": true305 },306 {307 "id": 32027,308 "content": "<extra_id_72>",309 "single_word": false,310 "lstrip": true,311 "rstrip": false,312 "normalized": true,313 "special": true314 },315 {316 "id": 32028,317 "content": "<extra_id_71>",318 "single_word": false,319 "lstrip": true,320 "rstrip": false,321 "normalized": true,322 "special": true323 },324 {325 "id": 32029,326 "content": "<extra_id_70>",327 "single_word": false,328 "lstrip": true,329 "rstrip": false,330 "normalized": true,331 "special": true332 },333 {334 "id": 32030,335 "content": "<extra_id_69>",336 "single_word": false,337 "lstrip": true,338 "rstrip": false,339 "normalized": true,340 "special": true341 },342 {343 "id": 32031,344 "content": "<extra_id_68>",345 "single_word": false,346 "lstrip": true,347 "rstrip": false,348 "normalized": true,349 "special": true350 },351 {352 "id": 32032,353 "content": "<extra_id_67>",354 "single_word": false,355 "lstrip": true,356 "rstrip": false,357 "normalized": true,358 "special": true359 },360 {361 "id": 32033,362 "content": "<extra_id_66>",363 "single_word": false,364 "lstrip": true,365 "rstrip": false,366 "normalized": true,367 "special": true368 },369 {370 "id": 32034,371 "content": "<extra_id_65>",372 "single_word": false,373 "lstrip": true,374 "rstrip": false,375 "normalized": true,376 "special": true377 },378 {379 "id": 32035,380 "content": "<extra_id_64>",381 "single_word": false,382 "lstrip": true,383 "rstrip": false,384 "normalized": true,385 "special": true386 },387 {388 "id": 32036,389 "content": "<extra_id_63>",390 "single_word": false,391 "lstrip": true,392 "rstrip": false,393 "normalized": true,394 "special": true395 },396 {397 "id": 32037,398 "content": "<extra_id_62>",399 "single_word": false,400 "lstrip": true,401 "rstrip": false,402 "normalized": true,403 "special": true404 },405 {406 "id": 32038,407 "content": "<extra_id_61>",408 "single_word": false,409 "lstrip": true,410 "rstrip": false,411 "normalized": true,412 "special": true413 },414 {415 "id": 32039,416 "content": "<extra_id_60>",417 "single_word": false,418 "lstrip": true,419 "rstrip": false,420 "normalized": true,421 "special": true422 },423 {424 "id": 32040,425 "content": "<extra_id_59>",426 "single_word": false,427 "lstrip": true,428 "rstrip": false,429 "normalized": true,430 "special": true431 },432 {433 "id": 32041,434 "content": "<extra_id_58>",435 "single_word": false,436 "lstrip": true,437 "rstrip": false,438 "normalized": true,439 "special": true440 },441 {442 "id": 32042,443 "content": "<extra_id_57>",444 "single_word": false,445 "lstrip": true,446 "rstrip": false,447 "normalized": true,448 "special": true449 },450 {451 "id": 32043,452 "content": "<extra_id_56>",453 "single_word": false,454 "lstrip": true,455 "rstrip": false,456 "normalized": true,457 "special": true458 },459 {460 "id": 32044,461 "content": "<extra_id_55>",462 "single_word": false,463 "lstrip": true,464 "rstrip": false,465 "normalized": true,466 "special": true467 },468 {469 "id": 32045,470 "content": "<extra_id_54>",471 "single_word": false,472 "lstrip": true,473 "rstrip": false,474 "normalized": true,475 "special": true476 },477 {478 "id": 32046,479 "content": "<extra_id_53>",480 "single_word": false,481 "lstrip": true,482 "rstrip": false,483 "normalized": true,484 "special": true485 },486 {487 "id": 32047,488 "content": "<extra_id_52>",489 "single_word": false,490 "lstrip": true,491 "rstrip": false,492 "normalized": true,493 "special": true494 },495 {496 "id": 32048,497 "content": "<extra_id_51>",498 "single_word": false,499 "lstrip": true,500 "rstrip": false,501 "normalized": true,502 "special": true503 },504 {505 "id": 32049,506 "content": "<extra_id_50>",507 "single_word": false,508 "lstrip": true,509 "rstrip": false,510 "normalized": true,511 "special": true512 },513 {514 "id": 32050,515 "content": "<extra_id_49>",516 "single_word": false,517 "lstrip": true,518 "rstrip": false,519 "normalized": true,520 "special": true521 },522 {523 "id": 32051,524 "content": "<extra_id_48>",525 "single_word": false,526 "lstrip": true,527 "rstrip": false,528 "normalized": true,529 "special": true530 },531 {532 "id": 32052,533 "content": "<extra_id_47>",534 "single_word": false,535 "lstrip": true,536 "rstrip": false,537 "normalized": true,538 "special": true539 },540 {541 "id": 32053,542 "content": "<extra_id_46>",543 "single_word": false,544 "lstrip": true,545 "rstrip": false,546 "normalized": true,547 "special": true548 },549 {550 "id": 32054,551 "content": "<extra_id_45>",552 "single_word": false,553 "lstrip": true,554 "rstrip": false,555 "normalized": true,556 "special": true557 },558 {559 "id": 32055,560 "content": "<extra_id_44>",561 "single_word": false,562 "lstrip": true,563 "rstrip": false,564 "normalized": true,565 "special": true566 },567 {568 "id": 32056,569 "content": "<extra_id_43>",570 "single_word": false,571 "lstrip": true,572 "rstrip": false,573 "normalized": true,574 "special": true575 },576 {577 "id": 32057,578 "content": "<extra_id_42>",579 "single_word": false,580 "lstrip": true,581 "rstrip": false,582 "normalized": true,583 "special": true584 },585 {586 "id": 32058,587 "content": "<extra_id_41>",588 "single_word": false,589 "lstrip": true,590 "rstrip": false,591 "normalized": true,592 "special": true593 },594 {595 "id": 32059,596 "content": "<extra_id_40>",597 "single_word": false,598 "lstrip": true,599 "rstrip": false,600 "normalized": true,601 "special": true602 },603 {604 "id": 32060,605 "content": "<extra_id_39>",606 "single_word": false,607 "lstrip": true,608 "rstrip": false,609 "normalized": true,610 "special": true611 },612 {613 "id": 32061,614 "content": "<extra_id_38>",615 "single_word": false,616 "lstrip": true,617 "rstrip": false,618 "normalized": true,619 "special": true620 },621 {622 "id": 32062,623 "content": "<extra_id_37>",624 "single_word": false,625 "lstrip": true,626 "rstrip": false,627 "normalized": true,628 "special": true629 },630 {631 "id": 32063,632 "content": "<extra_id_36>",633 "single_word": false,634 "lstrip": true,635 "rstrip": false,636 "normalized": true,637 "special": true638 },639 {640 "id": 32064,641 "content": "<extra_id_35>",642 "single_word": false,643 "lstrip": true,644 "rstrip": false,645 "normalized": true,646 "special": true647 },648 {649 "id": 32065,650 "content": "<extra_id_34>",651 "single_word": false,652 "lstrip": true,653 "rstrip": false,654 "normalized": true,655 "special": true656 },657 {658 "id": 32066,659 "content": "<extra_id_33>",660 "single_word": false,661 "lstrip": true,662 "rstrip": false,663 "normalized": true,664 "special": true665 },666 {667 "id": 32067,668 "content": "<extra_id_32>",669 "single_word": false,670 "lstrip": true,671 "rstrip": false,672 "normalized": true,673 "special": true674 },675 {676 "id": 32068,677 "content": "<extra_id_31>",678 "single_word": false,679 "lstrip": true,680 "rstrip": false,681 "normalized": true,682 "special": true683 },684 {685 "id": 32069,686 "content": "<extra_id_30>",687 "single_word": false,688 "lstrip": true,689 "rstrip": false,690 "normalized": true,691 "special": true692 },693 {694 "id": 32070,695 "content": "<extra_id_29>",696 "single_word": false,697 "lstrip": true,698 "rstrip": false,699 "normalized": true,700 "special": true701 },702 {703 "id": 32071,704 "content": "<extra_id_28>",705 "single_word": false,706 "lstrip": true,707 "rstrip": false,708 "normalized": true,709 "special": true710 },711 {712 "id": 32072,713 "content": "<extra_id_27>",714 "single_word": false,715 "lstrip": true,716 "rstrip": false,717 "normalized": true,718 "special": true719 },720 {721 "id": 32073,722 "content": "<extra_id_26>",723 "single_word": false,724 "lstrip": true,725 "rstrip": false,726 "normalized": true,727 "special": true728 },729 {730 "id": 32074,731 "content": "<extra_id_25>",732 "single_word": false,733 "lstrip": true,734 "rstrip": false,735 "normalized": true,736 "special": true737 },738 {739 "id": 32075,740 "content": "<extra_id_24>",741 "single_word": false,742 "lstrip": true,743 "rstrip": false,744 "normalized": true,745 "special": true746 },747 {748 "id": 32076,749 "content": "<extra_id_23>",750 "single_word": false,751 "lstrip": true,752 "rstrip": false,753 "normalized": true,754 "special": true755 },756 {757 "id": 32077,758 "content": "<extra_id_22>",759 "single_word": false,760 "lstrip": true,761 "rstrip": false,762 "normalized": true,763 "special": true764 },765 {766 "id": 32078,767 "content": "<extra_id_21>",768 "single_word": false,769 "lstrip": true,770 "rstrip": false,771 "normalized": true,772 "special": true773 },774 {775 "id": 32079,776 "content": "<extra_id_20>",777 "single_word": false,778 "lstrip": true,779 "rstrip": false,780 "normalized": true,781 "special": true782 },783 {784 "id": 32080,785 "content": "<extra_id_19>",786 "single_word": false,787 "lstrip": true,788 "rstrip": false,789 "normalized": true,790 "special": true791 },792 {793 "id": 32081,794 "content": "<extra_id_18>",795 "single_word": false,796 "lstrip": true,797 "rstrip": false,798 "normalized": true,799 "special": true800 },801 {802 "id": 32082,803 "content": "<extra_id_17>",804 "single_word": false,805 "lstrip": true,806 "rstrip": false,807 "normalized": true,808 "special": true809 },810 {811 "id": 32083,812 "content": "<extra_id_16>",813 "single_word": false,814 "lstrip": true,815 "rstrip": false,816 "normalized": true,817 "special": true818 },819 {820 "id": 32084,821 "content": "<extra_id_15>",822 "single_word": false,823 "lstrip": true,824 "rstrip": false,825 "normalized": true,826 "special": true827 },828 {829 "id": 32085,830 "content": "<extra_id_14>",831 "single_word": false,832 "lstrip": true,833 "rstrip": false,834 "normalized": true,835 "special": true836 },837 {838 "id": 32086,839 "content": "<extra_id_13>",840 "single_word": false,841 "lstrip": true,842 "rstrip": false,843 "normalized": true,844 "special": true845 },846 {847 "id": 32087,848 "content": "<extra_id_12>",849 "single_word": false,850 "lstrip": true,851 "rstrip": false,852 "normalized": true,853 "special": true854 },855 {856 "id": 32088,857 "content": "<extra_id_11>",858 "single_word": false,859 "lstrip": true,860 "rstrip": false,861 "normalized": true,862 "special": true863 },864 {865 "id": 32089,866 "content": "<extra_id_10>",867 "single_word": false,868 "lstrip": true,869 "rstrip": false,870 "normalized": true,871 "special": true872 },873 {874 "id": 32090,875 "content": "<extra_id_9>",876 "single_word": false,877 "lstrip": true,878 "rstrip": false,879 "normalized": true,880 "special": true881 },882 {883 "id": 32091,884 "content": "<extra_id_8>",885 "single_word": false,886 "lstrip": true,887 "rstrip": false,888 "normalized": true,889 "special": true890 },891 {892 "id": 32092,893 "content": "<extra_id_7>",894 "single_word": false,895 "lstrip": true,896 "rstrip": false,897 "normalized": true,898 "special": true899 },900 {901 "id": 32093,902 "content": "<extra_id_6>",903 "single_word": false,904 "lstrip": true,905 "rstrip": false,906 "normalized": true,907 "special": true908 },909 {910 "id": 32094,911 "content": "<extra_id_5>",912 "single_word": false,913 "lstrip": true,914 "rstrip": false,915 "normalized": true,916 "special": true917 },918 {919 "id": 32095,920 "content": "<extra_id_4>",921 "single_word": false,922 "lstrip": true,923 "rstrip": false,924 "normalized": true,925 "special": true926 },927 {928 "id": 32096,929 "content": "<extra_id_3>",930 "single_word": false,931 "lstrip": true,932 "rstrip": false,933 "normalized": true,934 "special": true935 },936 {937 "id": 32097,938 "content": "<extra_id_2>",939 "single_word": false,940 "lstrip": true,941 "rstrip": false,942 "normalized": true,943 "special": true944 },945 {946 "id": 32098,947 "content": "<extra_id_1>",948 "single_word": false,949 "lstrip": true,950 "rstrip": false,951 "normalized": true,952 "special": true953 },954 {955 "id": 32099,956 "content": "<extra_id_0>",957 "single_word": false,958 "lstrip": true,959 "rstrip": false,960 "normalized": true,961 "special": true962 }963 ],964 "normalizer": null,965 "pre_tokenizer": {966 "type": "ByteLevel",967 "add_prefix_space": false,968 "trim_offsets": true,969 "use_regex": true970 },971 "post_processor": {972 "type": "RobertaProcessing",973 "sep": [974 "</s>",975 2976 ],977 "cls": [978 "<s>",979 1980 ],981 "trim_offsets": true,982 "add_prefix_space": false983 },984 "decoder": {985 "type": "ByteLevel",986 "add_prefix_space": true,987 "trim_offsets": true,988 "use_regex": true989 },990 "model": {991 "type": "BPE",992 "dropout": null,993 "unk_token": null,994 "continuing_subword_prefix": "",995 "end_of_word_suffix": "",996 "fuse_unk": false,997 "vocab": {998 "<pad>": 0,999 "<s>": 1,1000 "</s>": 2,1001 "<unk>": 3,1002 "<mask>": 4,1003 "!": 5,1004 "\"": 6,1005 "#": 7,1006 "$": 8,1007 "%": 9,1008 "&": 10,1009 "'": 11,1010 "(": 12,1011 ")": 13,1012 "*": 14,1013 "+": 15,1014 ",": 16,1015 "-": 17,1016 ".": 18,1017 "/": 19,1018 "0": 20,1019 "1": 21,1020 "2": 22,1021 "3": 23,1022 "4": 24,1023 "5": 25,1024 "6": 26,1025 "7": 27,1026 "8": 28,1027 "9": 29,1028 ":": 30,1029 ";": 31,1030 "<": 32,1031 "=": 33,1032 ">": 34,1033 "?": 35,1034 "@": 36,1035 "A": 37,1036 "B": 38,1037 "C": 39,1038 "D": 40,1039 "E": 41,1040 "F": 42,1041 "G": 43,1042 "H": 44,1043 "I": 45,1044 "J": 46,1045 "K": 47,1046 "L": 48,1047 "M": 49,1048 "N": 50,1049 "O": 51,1050 "P": 52,1051 "Q": 53,1052 "R": 54,1053 "S": 55,1054 "T": 56,1055 "U": 57,1056 "V": 58,1057 "W": 59,1058 "X": 60,1059 "Y": 61,1060 "Z": 62,1061 "[": 63,1062 "\\": 64,1063 "]": 65,1064 "^": 66,1065 "_": 67,1066 "`": 68,1067 "a": 69,1068 "b": 70,1069 "c": 71,1070 "d": 72,1071 "e": 73,1072 "f": 74,1073 "g": 75,1074 "h": 76,1075 "i": 77,1076 "j": 78,1077 "k": 79,1078 "l": 80,1079 "m": 81,1080 "n": 82,1081 "o": 83,1082 "p": 84,1083 "q": 85,1084 "r": 86,1085 "s": 87,1086 "t": 88,1087 "u": 89,1088 "v": 90,1089 "w": 91,1090 "x": 92,1091 "y": 93,1092 "z": 94,1093 "{": 95,1094 "|": 96,1095 "}": 97,1096 "~": 98,1097 "¡": 99,1098 "¢": 100,1099 "£": 101,1100 "¤": 102,1101 "¥": 103,1102 "¦": 104,1103 "§": 105,1104 "¨": 106,1105 "©": 107,1106 "ª": 108,1107 "«": 109,1108 "¬": 110,1109 "®": 111,1110 "¯": 112,1111 "°": 113,1112 "±": 114,1113 "²": 115,1114 "³": 116,1115 "´": 117,1116 "µ": 118,1117 "¶": 119,1118 "·": 120,1119 "¸": 121,1120 "¹": 122,1121 "º": 123,1122 "»": 124,1123 "¼": 125,1124 "½": 126,1125 "¾": 127,1126 "¿": 128,1127 "À": 129,1128 "Á": 130,1129 "Â": 131,1130 "Ã": 132,1131 "Ä": 133,1132 "Å": 134,1133 "Æ": 135,1134 "Ç": 136,1135 "È": 137,1136 "É": 138,1137 "Ê": 139,1138 "Ë": 140,1139 "Ì": 141,1140 "Í": 142,1141 "Î": 143,1142 "Ï": 144,1143 "Ð": 145,1144 "Ñ": 146,1145 "Ò": 147,1146 "Ó": 148,1147 "Ô": 149,1148 "Õ": 150,1149 "Ö": 151,1150 "×": 152,1151 "Ø": 153,1152 "Ù": 154,1153 "Ú": 155,1154 "Û": 156,1155 "Ü": 157,1156 "Ý": 158,1157 "Þ": 159,1158 "ß": 160,1159 "à": 161,1160 "á": 162,1161 "â": 163,1162 "ã": 164,1163 "ä": 165,1164 "å": 166,1165 "æ": 167,1166 "ç": 168,1167 "è": 169,1168 "é": 170,1169 "ê": 171,1170 "ë": 172,1171 "ì": 173,1172 "í": 174,1173 "î": 175,1174 "ï": 176,1175 "ð": 177,1176 "ñ": 178,1177 "ò": 179,1178 "ó": 180,1179 "ô": 181,1180 "õ": 182,1181 "ö": 183,1182 "÷": 184,1183 "ø": 185,1184 "ù": 186,1185 "ú": 187,1186 "û": 188,1187 "ü": 189,1188 "ý": 190,1189 "þ": 191,1190 "ÿ": 192,1191 "Ā": 193,1192 "ā": 194,1193 "Ă": 195,1194 "ă": 196,1195 "Ą": 197,1196 "ą": 198,1197 "Ć": 199,1198 "ć": 200,1199 "Ĉ": 201,1200 "ĉ": 202,