hf-internal-testing/tokenization_test_data

1[2  {3    "TestClass": "AlbertTokenizationTest",4    "sp_base": false,5    "sequence": "I was born in 92000, and this is falsé.",6    "tokens": [7      "▁i",8      "▁was",9      "▁born",10      "▁in",11      "▁9",12      "2000",13      ",",14      "▁and",15      "▁this",16      "▁is",17      "▁false",18      "."19    ],20    "encoded": [21      31,22      23,23      386,24      19,25      561,26      3050,27      15,28      17,29      48,30      25,31      4997,32      933    ],34    "encoded_special": [35      2,36      31,37      23,38      386,39      19,40      561,41      3050,42      15,43      17,44      48,45      25,46      4997,47      9,48      349    ],50    "params": {},51    "params_encode": {}52  },53  {54    "TestClass": "BertTokenizationTest",55    "sp_base": false,56    "sequence": "UNwantéd,running",57    "tokens": [58      "un",59      "##want",60      "##ed",61      ",",62      "runn",63      "##ing"64    ],65    "encoded": [66      9,67      6,68      7,69      12,70      10,71      1172    ],73    "encoded_special": [74      1,75      9,76      6,77      7,78      12,79      10,80      11,81      282    ],83    "params": {},84    "params_encode": {}85  },86  {87    "TestClass": "BertTokenizationTest",88    "sp_base": false,89    "sequence": "UNwantéd,running",90    "tokens": [91      "un",92      "##want",93      "##ed",94      ",",95      "runn",96      "##ing"97    ],98    "encoded": [99      9,100      6,101      7,102      12,103      10,104      11105    ],106    "encoded_special": [107      1,108      9,109      6,110      7,111      12,112      10,113      11,114      2115    ],116    "params": {117      "do_lower_case": true118    },119    "params_encode": {}120  },121  {122    "TestClass": "BigBirdPegasusTokenizationTest",123    "sp_base": true,124    "sequence": "This is a test",125    "tokens": [126      "▁This",127      "▁is",128      "▁a",129      "▁",130      "t",131      "est"132    ],133    "encoded": [134      288,135      46,136      9,137      3,138      12,139      390140    ],141    "encoded_special": [142      288,143      46,144      9,145      3,146      12,147      390,148      1149    ],150    "params": {},151    "params_encode": {}152  },153  {154    "TestClass": "BigBirdTokenizationTest",155    "sp_base": false,156    "sequence": "I was born in 92000, and this is falsé.",157    "tokens": [158      "▁I",159      "▁was",160      "▁b",161      "or",162      "n",163      "▁in",164      "▁",165      "9",166      "2",167      "0",168      "0",169      "0",170      ",",171      "▁and",172      "▁this",173      "▁is",174      "▁f",175      "al",176      "s",177      "é",178      "."179    ],180    "encoded": [181      8,182      21,183      84,184      55,185      24,186      19,187      7,188      0,189      602,190      347,191      347,192      347,193      3,194      12,195      66,196      46,197      72,198      80,199      6,200      0,201      4202    ],203    "encoded_special": [204      1002,205      8,206      21,207      84,208      55,209      24,210      19,211      7,212      0,213      602,214      347,215      347,216      347,217      3,218      12,219      66,220      46,221      72,222      80,223      6,224      0,225      4,226      1000227    ],228    "params": {},229    "params_encode": {}230  },231  {232    "TestClass": "CLIPTokenizationTest",233    "sp_base": false,234    "sequence": "lower newer",235    "tokens": [236      "lo",237      "w",238      "er</w>",239      "n",240      "e",241      "w",242      "er</w>"243    ],244    "encoded": [245      10,246      2,247      16,248      9,249      3,250      2,251      16252    ],253    "encoded_special": [254      21,255      10,256      2,257      16,258      9,259      3,260      2,261      16,262      22263    ],264    "params": {},265    "params_encode": {}266  },267  {268    "TestClass": "CamembertTokenizationTest",269    "sp_base": true,270    "sequence": "I was born in 92000, and this is falsé.",271    "tokens": [272      "▁I",273      "▁was",274      "▁b",275      "or",276      "n",277      "▁in",278      "▁",279      "9",280      "2",281      "0",282      "0",283      "0",284      ",",285      "▁and",286      "▁this",287      "▁is",288      "▁f",289      "al",290      "s",291      "é",292      "."293    ],294    "encoded": [295      12,296      25,297      88,298      59,299      28,300      23,301      11,302      3,303      606,304      351,305      351,306      351,307      7,308      16,309      70,310      50,311      76,312      84,313      10,314      3,315      8316    ],317    "encoded_special": [318      5,319      12,320      25,321      88,322      59,323      28,324      23,325      11,326      3,327      606,328      351,329      351,330      351,331      7,332      16,333      70,334      50,335      76,336      84,337      10,338      3,339      8,340      6341    ],342    "params": {},343    "params_encode": {}344  },345  {346    "TestClass": "CodeGenTokenizationTest",347    "sp_base": false,348    "sequence": "lower newer",349    "tokens": [350      "Ġlow",351      "er",352      "Ġ",353      "n",354      "e",355      "w",356      "er"357    ],358    "encoded": [359      14,360      15,361      10,362      9,363      3,364      2,365      15366    ],367    "encoded_special": [368      14,369      15,370      10,371      9,372      3,373      2,374      15375    ],376    "params": {377      "add_prefix_space": true378    },379    "params_encode": {}380  },381  {382    "TestClass": "DPRContextEncoderTokenizationTest",383    "sp_base": false,384    "sequence": "UNwantéd,running",385    "tokens": [386      "un",387      "##want",388      "##ed",389      ",",390      "runn",391      "##ing"392    ],393    "encoded": [394      9,395      6,396      7,397      12,398      10,399      11400    ],401    "encoded_special": [402      1,403      9,404      6,405      7,406      12,407      10,408      11,409      2410    ],411    "params": {},412    "params_encode": {}413  },414  {415    "TestClass": "DPRQuestionEncoderTokenizationTest",416    "sp_base": false,417    "sequence": "UNwantéd,running",418    "tokens": [419      "un",420      "##want",421      "##ed",422      ",",423      "runn",424      "##ing"425    ],426    "encoded": [427      9,428      6,429      7,430      12,431      10,432      11433    ],434    "encoded_special": [435      1,436      9,437      6,438      7,439      12,440      10,441      11,442      2443    ],444    "params": {},445    "params_encode": {}446  },447  {448    "TestClass": "DPRReaderTokenizationTest",449    "sp_base": false,450    "sequence": "UNwantéd,running",451    "tokens": [452      "un",453      "##want",454      "##ed",455      ",",456      "runn",457      "##ing"458    ],459    "encoded": [460      9,461      6,462      7,463      12,464      10,465      11466    ],467    "encoded_special": [468      1,469      9,470      6,471      7,472      12,473      10,474      11,475      2476    ],477    "params": {},478    "params_encode": {}479  },480  {481    "TestClass": "DebertaTokenizationTest",482    "sp_base": false,483    "sequence": "lower newer",484    "tokens": [485      "l",486      "o",487      "w",488      "er",489      "Ġ",490      "n",491      "e",492      "w",493      "er"494    ],495    "encoded": [496      0,497      1,498      2,499      15,500      10,501      9,502      3,503      2,504      15505    ],506    "encoded_special": [507      20,508      0,509      1,510      2,511      15,512      10,513      9,514      3,515      2,516      15,517      21518    ],519    "params": {},520    "params_encode": {}521  },522  {523    "TestClass": "DebertaV2TokenizationTest",524    "sp_base": false,525    "sequence": "I was born in 92000, and this is falsé!",526    "tokens": [527      "▁",528      "I",529      "▁was",530      "▁born",531      "▁in",532      "▁9",533      "2000",534      ",",535      "▁and",536      "▁this",537      "▁is",538      "▁fal",539      "s",540      "é",541      "!"542    ],543    "encoded": [544      13,545      1,546      23,547      386,548      19,549      561,550      3050,551      15,552      17,553      48,554      25,555      8256,556      18,557      1,558      187559    ],560    "encoded_special": [561      2,562      13,563      1,564      23,565      386,566      19,567      561,568      3050,569      15,570      17,571      48,572      25,573      8256,574      18,575      1,576      187,577      3578    ],579    "params": {},580    "params_encode": {}581  },582  {583    "TestClass": "DistilBertTokenizationTest",584    "sp_base": false,585    "sequence": "UNwantéd,running",586    "tokens": [587      "un",588      "##want",589      "##ed",590      ",",591      "runn",592      "##ing"593    ],594    "encoded": [595      9,596      6,597      7,598      12,599      10,600      11601    ],602    "encoded_special": [603      1,604      9,605      6,606      7,607      12,608      10,609      11,610      2611    ],612    "params": {},613    "params_encode": {}614  },615  {616    "TestClass": "ElectraTokenizationTest",617    "sp_base": false,618    "sequence": "UNwantéd,running",619    "tokens": [620      "un",621      "##want",622      "##ed",623      ",",624      "runn",625      "##ing"626    ],627    "encoded": [628      9,629      6,630      7,631      12,632      10,633      11634    ],635    "encoded_special": [636      1,637      9,638      6,639      7,640      12,641      10,642      11,643      2644    ],645    "params": {},646    "params_encode": {}647  },648  {649    "TestClass": "ElectraTokenizationTest",650    "sp_base": false,651    "sequence": "UNwantéd,running",652    "tokens": [653      "un",654      "##want",655      "##ed",656      ",",657      "runn",658      "##ing"659    ],660    "encoded": [661      9,662      6,663      7,664      12,665      10,666      11667    ],668    "encoded_special": [669      1,670      9,671      6,672      7,673      12,674      10,675      11,676      2677    ],678    "params": {679      "do_lower_case": true680    },681    "params_encode": {}682  },683  {684    "TestClass": "FNetTokenizationTest",685    "sp_base": false,686    "sequence": "I was born in 92000, and this is falsé.",687    "tokens": [688      "▁",689      "I",690      "▁was",691      "▁born",692      "▁in",693      "▁9",694      "2000",695      ",",696      "▁and",697      "▁this",698      "▁is",699      "▁fal",700      "s",701      "é",702      "."703    ],704    "encoded": [705      13,706      1,707      23,708      386,709      19,710      561,711      3050,712      15,713      17,714      48,715      25,716      8256,717      18,718      1,719      9720    ],721    "encoded_special": [722      2,723      13,724      1,725      23,726      386,727      19,728      561,729      3050,730      15,731      17,732      48,733      25,734      8256,735      18,736      1,737      9,738      3739    ],740    "params": {},741    "params_encode": {}742  },743  {744    "TestClass": "FunnelTokenizationTest",745    "sp_base": false,746    "sequence": "UNwantéd,running",747    "tokens": [748      "un",749      "##want",750      "##ed",751      ",",752      "runn",753      "##ing"754    ],755    "encoded": [756      7,757      4,758      5,759      10,760      8,761      9762    ],763    "encoded_special": [764      1,765      7,766      4,767      5,768      10,769      8,770      9,771      2772    ],773    "params": {},774    "params_encode": {}775  },776  {777    "TestClass": "GPT2TokenizationTest",778    "sp_base": false,779    "sequence": "lower newer",780    "tokens": [781      "Ġlow",782      "er",783      "Ġ",784      "n",785      "e",786      "w",787      "er"788    ],789    "encoded": [790      14,791      15,792      10,793      9,794      3,795      2,796      15797    ],798    "encoded_special": [799      14,800      15,801      10,802      9,803      3,804      2,805      15806    ],807    "params": {808      "add_prefix_space": true809    },810    "params_encode": {}811  },812  {813    "TestClass": "HerbertTokenizationTest",814    "sp_base": false,815    "sequence": "lower,newer",816    "tokens": [817      "low",818      "er</w>",819      ",</w>",820      "n",821      "e",822      "w",823      "er</w>"824    ],825    "encoded": [826      16,827      17,828      22,829      11,830      5,831      4,832      17833    ],834    "encoded_special": [835      0,836      16,837      17,838      22,839      11,840      5,841      4,842      17,843      1844    ],845    "params": {},846    "params_encode": {}847  },848  {849    "TestClass": "LayoutLMTokenizationTest",850    "sp_base": false,851    "sequence": "UNwantéd,running",852    "tokens": [853      "un",854      "##want",855      "##ed",856      ",",857      "runn",858      "##ing"859    ],860    "encoded": [861      7,862      4,863      5,864      10,865      8,866      9867    ],868    "encoded_special": [869      1,870      7,871      4,872      5,873      10,874      8,875      9,876      2877    ],878    "params": {},879    "params_encode": {}880  },881  {882    "TestClass": "LayoutLMv2TokenizationTest",883    "sp_base": false,884    "sequence": null,885    "tokens": null,886    "encoded": [887      10,888      11,889      12,890      13891    ],892    "encoded_special": [893      1,894      10,895      11,896      12,897      13,898      2899    ],900    "params": {},901    "params_encode": {902      "text": [903        "a",904        "weirdly",905        "test"906      ],907      "boxes": [908        [909          423,910          237,911          440,912          251913        ],914        [915          427,916          272,917          441,918          287919        ],920        [921          419,922          115,923          437,924          129925        ]926      ]927    }928  },929  {930    "TestClass": "LayoutLMv3TokenizationTest",931    "sp_base": false,932    "sequence": null,933    "tokens": null,934    "encoded": [935      14,936      15,937      10,938      9,939      3,940      2,941      15942    ],943    "encoded_special": [944      20,945      14,946      15,947      10,948      9,949      3,950      2,951      15,952      21953    ],954    "params": {},955    "params_encode": {956      "text": [957        "lower",958        "newer"959      ],960      "boxes": [961        [962          423,963          237,964          440,965          251966        ],967        [968          427,969          272,970          441,971          287972        ]973      ]974    }975  },976  {977    "TestClass": "LayoutXLMTokenizationTest",978    "sp_base": true,979    "sequence": null,980    "tokens": null,981    "encoded": [982      11,983      113,984      159,985      17,986      39,987      171,988      383989    ],990    "encoded_special": [991      0,992      11,993      113,994      159,995      17,996      39,997      171,998      383,999      21000    ],1001    "params": {},1002    "params_encode": {1003      "text": [1004        "a",1005        "weirdly",1006        "test"1007      ],1008      "boxes": [1009        [1010          423,1011          237,1012          440,1013          2511014        ],1015        [1016          427,1017          272,1018          441,1019          2871020        ],1021        [1022          419,1023          115,1024          437,1025          1291026        ]1027      ]1028    }1029  },1030  {1031    "TestClass": "LongformerTokenizationTest",1032    "sp_base": false,1033    "sequence": "lower newer",1034    "tokens": [1035      "l",1036      "o",1037      "w",1038      "er",1039      "Ġ",1040      "n",1041      "e",1042      "w",1043      "er"1044    ],1045    "encoded": [1046      0,1047      1,1048      2,1049      15,1050      10,1051      9,1052      3,1053      2,1054      151055    ],1056    "encoded_special": [1057      20,1058      0,1059      1,1060      2,1061      15,1062      10,1063      9,1064      3,1065      2,1066      15,1067      211068    ],1069    "params": {},1070    "params_encode": {}1071  },1072  {1073    "TestClass": "LxmertTokenizationTest",1074    "sp_base": false,1075    "sequence": "I was born in 92000, and this is falsé.",1076    "tokens": [1077      "[UNK]",1078      "[UNK]",1079      "[UNK]",1080      "[UNK]",1081      "[UNK]",1082      ",",1083      "[UNK]",1084      "[UNK]",1085      "[UNK]",1086      "[UNK]",1087      "[UNK]"1088    ],1089    "encoded": [1090      0,1091      0,1092      0,1093      0,1094      0,1095      10,1096      0,1097      0,1098      0,1099      0,1100      01101    ],1102    "encoded_special": [1103      1,1104      0,1105      0,1106      0,1107      0,1108      0,1109      10,1110      0,1111      0,1112      0,1113      0,1114      0,1115      21116    ],1117    "params": {},1118    "params_encode": {}1119  },1120  {1121    "TestClass": "MBart50TokenizationTest",1122    "sp_base": true,1123    "sequence": "the I to a and of in was it me that be he for with my not is s you",1124    "tokens": [1125      "▁the",1126      "▁I",1127      "▁to",1128      "▁a",1129      "▁and",1130      "▁of",1131      "▁in",1132      "▁was",1133      "▁it",1134      "▁me",1135      "▁that",1136      "▁be",1137      "▁he",1138      "▁for",1139      "▁with",1140      "▁my",1141      "▁not",1142      "▁is",1143      "▁s",1144      "▁you"1145    ],1146    "encoded": [1147      6,1148      9,1149      10,1150      11,1151      13,1152      14,1153      20,1154      22,1155      26,1156      32,1157      35,1158      37,1159      38,1160      41,1161      42,1162      44,1163      45,1164      47,1165      48,1166      491167    ],1168    "encoded_special": [1169      1004,1170      6,1171      9,1172      10,1173      11,1174      13,1175      14,1176      20,1177      22,1178      26,1179      32,1180      35,1181      37,1182      38,1183      41,1184      42,1185      44,1186      45,1187      47,1188      48,1189      49,1190      21191    ],1192    "params": {},1193    "params_encode": {}1194  },1195  {1196    "TestClass": "MBartTokenizationTest",1197    "sp_base": true,1198    "sequence": "the I to a and of in was it me that be he for with my not is s you",1199    "tokens": [1200      "▁the",

Showing the first 1,200 of 3242 lines. Download the file for the rest.