placeholderlabs/exp-pool-commit-code-dolma2-tokenized
Locus EXP Commit Code - Dolma 2 tokenized Pretokenized experiment pool for reproducible proxy-training runs. MANIFEST.json is the authoritative schema, provenance, checksums, and train/holdout assignment. shards/shard-NNNNN/tokens.bin stores little-endian int32 token IDs. offsets.bin stores little-endian int64 document boundaries. index.parquet stores document IDs, offsets, and compact filter fields. metadata.parquet stores complete source metadata and is downloaded only for… See the full description on the dataset page: https://huggingface.co/datasets/placeholderlabs/exp-pool-commit-code-dolma2-tokenized.
033
1{2 "candidate_store_codec": "zlib-level-1",3 "candidate_store_schema_version": 1,4 "commit_id_conflict_policy": "keep-first-in-seeded-shard-row-order",5 "document_representation": "hot typed index plus complete cold provenance/source metadata and token offsets",6 "documents": 5116544,7 "field_extractions": {8 "ai_authorship_signal": "annotations.possible_ai_authorship.signal",9 "commit_score": "annotations.commit_quality_score.commit_score",10 "primary_language": "annotations.programming_languages.primary_language",11 "quality_band": "annotations.quality_band",12 "repository_name": "meta.repository_id",13 "tests_touched": "annotations.tests_touched.touched",14 "verification_state": "annotations.verification_state"15 },16 "generator_packages": {17 "huggingface-hub": "1.15.0",18 "numpy": "2.5.2",19 "pyarrow": "23.0.1",20 "tokenizers": "0.23.1"21 },22 "holdout_raw_paths": [23 "data/train-00127-of-00128.parquet"24 ],25 "index_schema": [26 {27 "name": "start",28 "nullable": false,29 "type": "int64"30 },31 {32 "name": "end",33 "nullable": false,34 "type": "int64"35 },36 {37 "name": "document_id",38 "nullable": false,39 "type": "string"40 },41 {42 "name": "source_name",43 "nullable": false,44 "type": "string"45 },46 {47 "name": "source_stratum",48 "nullable": false,49 "type": "string"50 },51 {52 "name": "utf8_bytes",53 "nullable": false,54 "type": "int64"55 },56 {57 "name": "conservative_estimated_tokens",58 "nullable": false,59 "type": "int64"60 },61 {62 "name": "document_order_key",63 "nullable": false,64 "type": "uint64"65 },66 {67 "name": "quality_band",68 "nullable": true,69 "type": "string"70 },71 {72 "name": "tests_touched",73 "nullable": true,74 "type": "bool"75 },76 {77 "name": "ai_authorship_signal",78 "nullable": true,79 "type": "string"80 },81 {82 "name": "commit_score",83 "nullable": true,84 "type": "int32"85 },86 {87 "name": "primary_language",88 "nullable": true,89 "type": "string"90 },91 {92 "name": "verification_state",93 "nullable": true,94 "type": "string"95 },96 {97 "name": "repository_name",98 "nullable": true,99 "type": "string"100 },101 {102 "name": "target_start_token",103 "nullable": false,104 "type": "int32"105 }106 ],107 "metadata_schema": [108 {109 "name": "document_id",110 "nullable": false,111 "type": "string"112 },113 {114 "name": "provenance_json",115 "nullable": false,116 "type": "string"117 },118 {119 "name": "source_metadata_json",120 "nullable": false,121 "type": "string"122 }123 ],124 "must_fit_sequence": true,125 "pool": "commit_code",126 "raw_source": {127 "repository": "placeholderlabs/exp-pool-commit-code-raw",128 "revision": "21b0b1258457843e08d93a47b07405004e60227a"129 },130 "rendered_bytes_upper_bound": 1048320,131 "schema": "locus.tokenized-pool/v2",132 "sequence_length": 4096,133 "shards": [134 {135 "document_ids_sha256": "e823a873f68be19476f5f2b0f0886833d1bbb6c9de2620ff01c786cf3d7ea6da",136 "documents": 38288,137 "files": {138 "index": {139 "bytes": 3592396,140 "path": "shards/shard-00000/index.parquet",141 "sha256": "7e396891ce61460179248d4b18447b4cfcc2b4e84d33cd579a40acf1ad12948a"142 },143 "metadata": {144 "bytes": 24169133,145 "path": "shards/shard-00000/metadata.parquet",146 "sha256": "6f26954dd91ca2b9561d144f72986cff6e53a047b7ab09b01b607a65a767de7b"147 },148 "offsets": {149 "bytes": 306312,150 "path": "shards/shard-00000/offsets.bin",151 "sha256": "4949418c034d5f66f0114ef148816e09c9bc58b04c64f6994cb86fe2926f5c36"152 },153 "tokens": {154 "bytes": 134735636,155 "path": "shards/shard-00000/tokens.bin",156 "sha256": "b37a245c0d992f531783655deceb4c958e7f1014a7590a14ddfdf8832a89ff98"157 }158 },159 "partition": "train",160 "raw_path": "data/train-00000-of-00128.parquet",161 "shard_index": 0,162 "tokens": 33683909163 },164 {165 "document_ids_sha256": "f51950561b60a533bd2cd7f2d56137766822a9b0f6d8cb4f1bb3bd82f912502a",166 "documents": 38408,167 "files": {168 "index": {169 "bytes": 3602017,170 "path": "shards/shard-00001/index.parquet",171 "sha256": "c1d44b031525dfd9c4a223bdb9cdf7a08977d1ab7d4e3c6c011529ea5fe82a4e"172 },173 "metadata": {174 "bytes": 24287189,175 "path": "shards/shard-00001/metadata.parquet",176 "sha256": "1ba5b77d38e3700f9fe9f401aea84522c16fe2cb82ffe0475a34313a440d1f7c"177 },178 "offsets": {179 "bytes": 307272,180 "path": "shards/shard-00001/offsets.bin",181 "sha256": "5c2cf4161f4eab5fbec188c01e960b10e2499459d46390299efe69bd659e39fa"182 },183 "tokens": {184 "bytes": 133644956,185 "path": "shards/shard-00001/tokens.bin",186 "sha256": "aa555316de2a8919f512a00a4ee4a3afd67c0afbb302dfbec25fe331bb7dc174"187 }188 },189 "partition": "train",190 "raw_path": "data/train-00001-of-00128.parquet",191 "shard_index": 1,192 "tokens": 33411239193 },194 {195 "document_ids_sha256": "b1731b9e4a3a0080579909c2ea2fd510ba4ac28357635c02b83ce56a0849ff8e",196 "documents": 38273,197 "files": {198 "index": {199 "bytes": 3590504,200 "path": "shards/shard-00002/index.parquet",201 "sha256": "1435f0b1a564be1502a97b417b7d459c40faf809f3e756d1ee4f3b1728998bf8"202 },203 "metadata": {204 "bytes": 24122488,205 "path": "shards/shard-00002/metadata.parquet",206 "sha256": "f0828e9eaa74908cfd72e03fdff246af132eac8bce778a2597401d945dcdc99b"207 },208 "offsets": {209 "bytes": 306192,210 "path": "shards/shard-00002/offsets.bin",211 "sha256": "495fbefdb8c4785be0b877bd4d8ead3cf781210dcfb2129c0bc1d1e3903b9614"212 },213 "tokens": {214 "bytes": 133294104,215 "path": "shards/shard-00002/tokens.bin",216 "sha256": "848dc26ab9116e76a5f54373c1e9e9d799f82034f694705fbd40e0342554964d"217 }218 },219 "partition": "train",220 "raw_path": "data/train-00002-of-00128.parquet",221 "shard_index": 2,222 "tokens": 33323526223 },224 {225 "document_ids_sha256": "81361e1a0a38d66780231c900ef4310e47341b2029f897e7b0360dca678c6a6e",226 "documents": 38434,227 "files": {228 "index": {229 "bytes": 3606595,230 "path": "shards/shard-00003/index.parquet",231 "sha256": "1c46b14ec0172740db56cad0c38af0083b66b28a1b8f670173762d5687be4535"232 },233 "metadata": {234 "bytes": 24228101,235 "path": "shards/shard-00003/metadata.parquet",236 "sha256": "c3ff42836b0426fca3f81cdc1debf42525d93b330e49990988ed5aed0ac3bce1"237 },238 "offsets": {239 "bytes": 307480,240 "path": "shards/shard-00003/offsets.bin",241 "sha256": "952312ba341cdcf3949e654bb1604e81342995f71130683a4fbf03f59fd5c0ce"242 },243 "tokens": {244 "bytes": 133988616,245 "path": "shards/shard-00003/tokens.bin",246 "sha256": "d0fe6a6c77e1cb9be98af778a9315c3d02c277c62a342e2243c04d275bb75100"247 }248 },249 "partition": "train",250 "raw_path": "data/train-00003-of-00128.parquet",251 "shard_index": 3,252 "tokens": 33497154253 },254 {255 "document_ids_sha256": "c2ac366b982696b298330ba4c9236819cf2806a3123608c04c0501db2f6ef5a9",256 "documents": 38382,257 "files": {258 "index": {259 "bytes": 3596911,260 "path": "shards/shard-00004/index.parquet",261 "sha256": "b12cbfae1b340efc258b68cd16f06a09ec87acf5ce43d9d0cd150b318900b5f0"262 },263 "metadata": {264 "bytes": 24236457,265 "path": "shards/shard-00004/metadata.parquet",266 "sha256": "281840ac591421754ac30e4779c0eebeedf28444800cb4659deb3d5197ed8d6f"267 },268 "offsets": {269 "bytes": 307064,270 "path": "shards/shard-00004/offsets.bin",271 "sha256": "8007a05c79119c99fc2107d7b14be29c5090457b8bf955a28c63b7ec4601d740"272 },273 "tokens": {274 "bytes": 133400964,275 "path": "shards/shard-00004/tokens.bin",276 "sha256": "e4f017f4a8d62372ddc23b5b7d591cce21d36a900d038cbaf00979a89027e85a"277 }278 },279 "partition": "train",280 "raw_path": "data/train-00004-of-00128.parquet",281 "shard_index": 4,282 "tokens": 33350241283 },284 {285 "document_ids_sha256": "0caaa74354918afc8cab1a92783c1e71f917343e2e207caf3d9fbd6a9b53edb1",286 "documents": 38484,287 "files": {288 "index": {289 "bytes": 3612609,290 "path": "shards/shard-00005/index.parquet",291 "sha256": "0e17bced2b2d31c0577521b05d9b4cf457b2338adb514c070704423d344b6d81"292 },293 "metadata": {294 "bytes": 24434615,295 "path": "shards/shard-00005/metadata.parquet",296 "sha256": "3f39ad6bccfc848d5f6e3fd67e06a5bf562084617a773280a1d3846b7485faa5"297 },298 "offsets": {299 "bytes": 307880,300 "path": "shards/shard-00005/offsets.bin",301 "sha256": "d0a4812f2a4800f2abf13f3fa7b7ad11f25f942aa74d58bf9dba47128ae852a0"302 },303 "tokens": {304 "bytes": 134637880,305 "path": "shards/shard-00005/tokens.bin",306 "sha256": "9c3b289a2b7bedba6b48cb2c02e0f7ddb48996390f983a136d8ff626eed7f6a1"307 }308 },309 "partition": "train",310 "raw_path": "data/train-00005-of-00128.parquet",311 "shard_index": 5,312 "tokens": 33659470313 },314 {315 "document_ids_sha256": "03d2bb9cf88bd8adc97c6e2d930bd1dfa2627d407cc8a1a7398e6b195b61397c",316 "documents": 38177,317 "files": {318 "index": {319 "bytes": 3583141,320 "path": "shards/shard-00006/index.parquet",321 "sha256": "8c08e12da36803729d6dfc989482d06a79ad1f36f8ebaf5eef098a15dd070eb8"322 },323 "metadata": {324 "bytes": 24062987,325 "path": "shards/shard-00006/metadata.parquet",326 "sha256": "d91c82d7ac8032d9ed7f3d3fc8e380edeb54ba0106aa536c2bbea747f07603f6"327 },328 "offsets": {329 "bytes": 305424,330 "path": "shards/shard-00006/offsets.bin",331 "sha256": "669dc943419c626d867d5e1ceea28ba07092db48b7a727463c23212283d6e0f6"332 },333 "tokens": {334 "bytes": 132969828,335 "path": "shards/shard-00006/tokens.bin",336 "sha256": "31d9869f42e7a92aa9e2ea72da906993fdfa9bf0920f64ba4f7fec40dcb543df"337 }338 },339 "partition": "train",340 "raw_path": "data/train-00006-of-00128.parquet",341 "shard_index": 6,342 "tokens": 33242457343 },344 {345 "document_ids_sha256": "43eaa8ed2f465ec78b742d91ee7b8c047a8ea25e766e572132e9fae1b2f87a84",346 "documents": 38433,347 "files": {348 "index": {349 "bytes": 3606683,350 "path": "shards/shard-00007/index.parquet",351 "sha256": "511b7d5d6c8fae00ef90ae9927fa0855eeb2c9fc9ae8b425b55b5a1f9401ca83"352 },353 "metadata": {354 "bytes": 24225552,355 "path": "shards/shard-00007/metadata.parquet",356 "sha256": "0e637d3263a603d58ac7b0ae00f2ae7c28ff24704e12aabcb14e4b540dec3199"357 },358 "offsets": {359 "bytes": 307472,360 "path": "shards/shard-00007/offsets.bin",361 "sha256": "6f6b725164460e68ce9055926bc91c2ca5764a6fe604f0925d3ccfe5cfd520c7"362 },363 "tokens": {364 "bytes": 134708356,365 "path": "shards/shard-00007/tokens.bin",366 "sha256": "5b248bf5a2c15e4ba253a1d5d71c1652b48dfd1d09f54b053d79a2bed88a345f"367 }368 },369 "partition": "train",370 "raw_path": "data/train-00007-of-00128.parquet",371 "shard_index": 7,372 "tokens": 33677089373 },374 {375 "document_ids_sha256": "401d7fb5a119a4f79a4b77c0f64648cb743341b0b25c0df6e3ca0c36610b048f",376 "documents": 38616,377 "files": {378 "index": {379 "bytes": 3620920,380 "path": "shards/shard-00008/index.parquet",381 "sha256": "e79b80d26d2ef0cbaf858f744cc5c9920a4017846b2f1e8656e5167393ebbabb"382 },383 "metadata": {384 "bytes": 24273656,385 "path": "shards/shard-00008/metadata.parquet",386 "sha256": "4031da21d5fa364c225ddbada01dcf43fa29339dceffd1881be9c8a0392c701f"387 },388 "offsets": {389 "bytes": 308936,390 "path": "shards/shard-00008/offsets.bin",391 "sha256": "2550ac5afe794804d7af04812db8499885d043ac956de7e3d71ca9f0a23b44a9"392 },393 "tokens": {394 "bytes": 134243468,395 "path": "shards/shard-00008/tokens.bin",396 "sha256": "92dd03b26af1819735237faaf9f30c16ab7d726b5993bea62a0304833473541a"397 }398 },399 "partition": "train",400 "raw_path": "data/train-00008-of-00128.parquet",401 "shard_index": 8,402 "tokens": 33560867403 },404 {405 "document_ids_sha256": "f52217a0614648cd3880d48ee08028007e519f820fad24e1bc2ef829096b83d9",406 "documents": 38129,407 "files": {408 "index": {409 "bytes": 3574662,410 "path": "shards/shard-00009/index.parquet",411 "sha256": "344e51fc95d0ebc369faec1fe154ff2ec3a0f043fe94698e07d2f747e31a014e"412 },413 "metadata": {414 "bytes": 24411417,415 "path": "shards/shard-00009/metadata.parquet",416 "sha256": "9b3659b0b90b01b6753d0d85cf22ef81496630e28b1e94cd0894f45937eade98"417 },418 "offsets": {419 "bytes": 305040,420 "path": "shards/shard-00009/offsets.bin",421 "sha256": "ed392b870d19cbcbf303b676f9690fa4255147881139b477795cacdf5e6b3784"422 },423 "tokens": {424 "bytes": 132506512,425 "path": "shards/shard-00009/tokens.bin",426 "sha256": "463d54fc636205b97234d4afffd8805b53de47691d3ea5de1624363a21d4fc6b"427 }428 },429 "partition": "train",430 "raw_path": "data/train-00009-of-00128.parquet",431 "shard_index": 9,432 "tokens": 33126628433 },434 {435 "document_ids_sha256": "9e646d553886891a4863c800e5675dacd2d4e314791032e1d9df9c98f1f39ba0",436 "documents": 38400,437 "files": {438 "index": {439 "bytes": 3603978,440 "path": "shards/shard-00010/index.parquet",441 "sha256": "aa3e8812e72d22a9668431be7b98d42f78cda80356ac5c4ab22af7e34ef7480d"442 },443 "metadata": {444 "bytes": 24272832,445 "path": "shards/shard-00010/metadata.parquet",446 "sha256": "cb2cfdd249d108e4b15857dc08fd043226e7a5ebcbf7bac1d061a0ddbe7ced78"447 },448 "offsets": {449 "bytes": 307208,450 "path": "shards/shard-00010/offsets.bin",451 "sha256": "ac99641ab929b9bc9c42ab36e163a42d0d7d2fed8e266e4bcf9d534f3f089fd9"452 },453 "tokens": {454 "bytes": 134541840,455 "path": "shards/shard-00010/tokens.bin",456 "sha256": "bcea8e175ce237aae862ac4f0e4556cd8f874c064ac3eee7e5174951b6528e1e"457 }458 },459 "partition": "train",460 "raw_path": "data/train-00010-of-00128.parquet",461 "shard_index": 10,462 "tokens": 33635460463 },464 {465 "document_ids_sha256": "12068a140fbe1b28d29a5e8bde1a4336da7be7af6ba08cb704c389cad8f1863e",466 "documents": 38178,467 "files": {468 "index": {469 "bytes": 3580254,470 "path": "shards/shard-00011/index.parquet",471 "sha256": "c48c7cb0e55bdbfc60eaf9dc22a8d7d08aa3975b594cabd7098dbc67d9bada0c"472 },473 "metadata": {474 "bytes": 24000149,475 "path": "shards/shard-00011/metadata.parquet",476 "sha256": "bf1b46a2763fef1477489b9965a287f86aa761fda4bfedd7bf9cf9ad9553cf62"477 },478 "offsets": {479 "bytes": 305432,480 "path": "shards/shard-00011/offsets.bin",481 "sha256": "242ac8eaf201d20be045e87ce43d4e2de404f45f298e4e2aa6f472e5dfc6842e"482 },483 "tokens": {484 "bytes": 133187116,485 "path": "shards/shard-00011/tokens.bin",486 "sha256": "f2c0ac24d9a767079dcab814aa72c46893c9dbcea0e601d5e11f3d3759a4fe2b"487 }488 },489 "partition": "train",490 "raw_path": "data/train-00011-of-00128.parquet",491 "shard_index": 11,492 "tokens": 33296779493 },494 {495 "document_ids_sha256": "aef580180bb628a0973f1830781d058ffcca83ca2fd9cc89b223feeb3e20a671",496 "documents": 38734,497 "files": {498 "index": {499 "bytes": 3638182,500 "path": "shards/shard-00012/index.parquet",501 "sha256": "9a8850c234c113a15113ca29f3b748dab57a99e351da7456595c82807070ea6a"502 },503 "metadata": {504 "bytes": 24452377,505 "path": "shards/shard-00012/metadata.parquet",506 "sha256": "d0084d70e970154f08b3aebebba6b5f4afb8c330fa39352163f4c3efdb744d31"507 },508 "offsets": {509 "bytes": 309880,510 "path": "shards/shard-00012/offsets.bin",511 "sha256": "340a0fe75db7d020c2d4d1dfcdcca06d0fd76f80febf2d5bc3cd77b54602aff0"512 },513 "tokens": {514 "bytes": 136047580,515 "path": "shards/shard-00012/tokens.bin",516 "sha256": "af12f4ceedfeba389a06c0e8ea7157a95ebd44d581f64205d42204a1d90a97e2"517 }518 },519 "partition": "train",520 "raw_path": "data/train-00012-of-00128.parquet",521 "shard_index": 12,522 "tokens": 34011895523 },524 {525 "document_ids_sha256": "56a3d7b249166b7ba4c8c4c6e46eb0cb81647520ff8e7c693fed21301f1da17b",526 "documents": 38206,527 "files": {528 "index": {529 "bytes": 3581629,530 "path": "shards/shard-00013/index.parquet",531 "sha256": "55cc80a1744470afb9fec5a357d1375a1efa861e42c754f84cc5c3abf1293bfe"532 },533 "metadata": {534 "bytes": 24039589,535 "path": "shards/shard-00013/metadata.parquet",536 "sha256": "d3a4ca541b09845818621225e3b73cfc4893cc1be1597370dc805660fc6194c9"537 },538 "offsets": {539 "bytes": 305656,540 "path": "shards/shard-00013/offsets.bin",541 "sha256": "98274b7bb7fd3d24109ad5698c57e6ed41209d749e0d2c35cea6ccb0fe11a122"542 },543 "tokens": {544 "bytes": 131862840,545 "path": "shards/shard-00013/tokens.bin",546 "sha256": "a044e156d984e5c9baf6aa670a1770e9ea10e2b4cfdec2cf5fe709f5363121d2"547 }548 },549 "partition": "train",550 "raw_path": "data/train-00013-of-00128.parquet",551 "shard_index": 13,552 "tokens": 32965710553 },554 {555 "document_ids_sha256": "9021794d659c62564c18af7c9ff41ad788b366afcae050b50c9b0f07e3da722e",556 "documents": 38149,557 "files": {558 "index": {559 "bytes": 3576384,560 "path": "shards/shard-00014/index.parquet",561 "sha256": "cb7c6f303a4c869377fbda72e891ce4b8b1060899c9247f815b492fe1e713b21"562 },563 "metadata": {564 "bytes": 24032902,565 "path": "shards/shard-00014/metadata.parquet",566 "sha256": "a0a1b535e8f2dc62523c08d47974f4d4a7c413fc1ed2f9c7a147718b4009b1fe"567 },568 "offsets": {569 "bytes": 305200,570 "path": "shards/shard-00014/offsets.bin",571 "sha256": "87dab9879c54d6ee80627e0347189e6b198bf0d4f52f7ea8107a2f1007deff7f"572 },573 "tokens": {574 "bytes": 133348924,575 "path": "shards/shard-00014/tokens.bin",576 "sha256": "326f6bfb543fb4580a8df1d3417991b8cf98011ee4a50d793a7297ceb766e8e6"577 }578 },579 "partition": "train",580 "raw_path": "data/train-00014-of-00128.parquet",581 "shard_index": 14,582 "tokens": 33337231583 },584 {585 "document_ids_sha256": "409c77439e22cd8b9e4ed57bba9f77896cee9b41e05ad5714343fbcb37721fb5",586 "documents": 38408,587 "files": {588 "index": {589 "bytes": 3604590,590 "path": "shards/shard-00015/index.parquet",591 "sha256": "cb93b82e200b2efe3901859f04327ff999e518ee162b60c03ae4b267bcb82e69"592 },593 "metadata": {594 "bytes": 24324810,595 "path": "shards/shard-00015/metadata.parquet",596 "sha256": "d3c22eb2e290e3844a34ae3e903891137e89dd3d3511d32e3f654248b9f53acc"597 },598 "offsets": {599 "bytes": 307272,600 "path": "shards/shard-00015/offsets.bin",601 "sha256": "7f89f69eac559889a17da410033d3143aab911f9cdb6edf32865cd8c62d1bf65"602 },603 "tokens": {604 "bytes": 133761760,605 "path": "shards/shard-00015/tokens.bin",606 "sha256": "7c0217b1bd7305787aacdef9a83fa02a732b841690c502b20752e21142269110"607 }608 },609 "partition": "train",610 "raw_path": "data/train-00015-of-00128.parquet",611 "shard_index": 15,612 "tokens": 33440440613 },614 {615 "document_ids_sha256": "d4697c65a5426206413bcb94f34b9e4c6c1ce9f3f578d0057e1768abf1abbbf0",616 "documents": 38113,617 "files": {618 "index": {619 "bytes": 3572657,620 "path": "shards/shard-00016/index.parquet",621 "sha256": "c638ff064d511d0b8f32d3360b4cb8be008624aab1f7d257784089d415975a1b"622 },623 "metadata": {624 "bytes": 23985848,625 "path": "shards/shard-00016/metadata.parquet",626 "sha256": "6449df7deaf325b36010ab0b7a31b125187b2a2378f9bb91f5d73086634f8bd7"627 },628 "offsets": {629 "bytes": 304912,630 "path": "shards/shard-00016/offsets.bin",631 "sha256": "b13f0bf0b18961f99bfbc7e046d8d6ad4e777ecb2d2c47c3d4a1777d604a0fdc"632 },633 "tokens": {634 "bytes": 134102636,635 "path": "shards/shard-00016/tokens.bin",636 "sha256": "79f3d30038d997b1c72e0abd3cc4ff78b83b90fec40973f1a95481d2ae639612"637 }638 },639 "partition": "train",640 "raw_path": "data/train-00016-of-00128.parquet",641 "shard_index": 16,642 "tokens": 33525659643 },644 {645 "document_ids_sha256": "74cf021673c8fc38aba96a6fa848103c2243322e84f8e5ef61636da7513a1668",646 "documents": 38186,647 "files": {648 "index": {649 "bytes": 3583177,650 "path": "shards/shard-00017/index.parquet",651 "sha256": "63202ca9ee8c1a492ca15db2f75bb9f8d5b16a4a9f448cbd2e73f7ede0f2b685"652 },653 "metadata": {654 "bytes": 24109597,655 "path": "shards/shard-00017/metadata.parquet",656 "sha256": "97191e1fb1585cb32282ac6ef1eafa20e7f60af290fe924c189ceaf7b48e057d"657 },658 "offsets": {659 "bytes": 305496,660 "path": "shards/shard-00017/offsets.bin",661 "sha256": "60591dbed1f6a6a59c3e9e744d02c4e32a90b90238f000cb2a23fbf44b3cdda6"662 },663 "tokens": {664 "bytes": 133557856,665 "path": "shards/shard-00017/tokens.bin",666 "sha256": "f9ac2da278c159ffd04190535dbc9814d7f852c137519b6969596f26c4a9f668"667 }668 },669 "partition": "train",670 "raw_path": "data/train-00017-of-00128.parquet",671 "shard_index": 17,672 "tokens": 33389464673 },674 {675 "document_ids_sha256": "eb86cf21599939aeb8634952cba1c62d28e5751fa7deecc4d5aebef1d2c69321",676 "documents": 38465,677 "files": {678 "index": {679 "bytes": 3609556,680 "path": "shards/shard-00018/index.parquet",681 "sha256": "ccecb34474166a98b0fd4d309393afc1fcbd17f44a8ee05897ed91604eab51b0"682 },683 "metadata": {684 "bytes": 24227142,685 "path": "shards/shard-00018/metadata.parquet",686 "sha256": "d7792db1b5e415f0d73b23cf334cae316533f4eb1cd6f8f1747a87716c7ccdb6"687 },688 "offsets": {689 "bytes": 307728,690 "path": "shards/shard-00018/offsets.bin",691 "sha256": "b71c68c9552554fced80eb4628f366dcd8836cba215ac62d99132168e5805810"692 },693 "tokens": {694 "bytes": 134315576,695 "path": "shards/shard-00018/tokens.bin",696 "sha256": "f83d18cdf3fa14662924ec1bcbbb5ea5ad762526fe7d50b2e38952c676f9a8dc"697 }698 },699 "partition": "train",700 "raw_path": "data/train-00018-of-00128.parquet",701 "shard_index": 18,702 "tokens": 33578894703 },704 {705 "document_ids_sha256": "7ee5213cbcdba9984d3b89203c484aa8c72d84110137e29b4b4fe646d7b39277",706 "documents": 38370,707 "files": {708 "index": {709 "bytes": 3600232,710 "path": "shards/shard-00019/index.parquet",711 "sha256": "3b03f9c7f07fcb872afd7cc80aadcce027ad257ff43665624f44cc25649202e7"712 },713 "metadata": {714 "bytes": 24221733,715 "path": "shards/shard-00019/metadata.parquet",716 "sha256": "4323acc7e01db8c3e2671047cf28606913875f72b6bf283041559a3b4343eca1"717 },718 "offsets": {719 "bytes": 306968,720 "path": "shards/shard-00019/offsets.bin",721 "sha256": "9aa35190d9cd515917d073530726165e81627db53566680f3b893b598b36eb52"722 },723 "tokens": {724 "bytes": 134542052,725 "path": "shards/shard-00019/tokens.bin",726 "sha256": "7381c99c3c905430666ca2948157d39e6bdbcf3d1cbd02786f4754f162766df8"727 }728 },729 "partition": "train",730 "raw_path": "data/train-00019-of-00128.parquet",731 "shard_index": 19,732 "tokens": 33635513733 },734 {735 "document_ids_sha256": "eca408281d23e5d19b83b64097e192d4df75a4eef51e33107d03732397104271",736 "documents": 38387,737 "files": {738 "index": {739 "bytes": 3598188,740 "path": "shards/shard-00020/index.parquet",741 "sha256": "5425623f2298155bfff72ac9bd527157fb575ba1b9585a4131cdc5cc6012a970"742 },743 "metadata": {744 "bytes": 24190796,745 "path": "shards/shard-00020/metadata.parquet",746 "sha256": "552ebd050d021ad3e7031eb704910309a4feb9e7b129ee34d4ff8de1ad1cd8f6"747 },748 "offsets": {749 "bytes": 307104,750 "path": "shards/shard-00020/offsets.bin",751 "sha256": "55be088b90266400d9740be5ced34336a0d60c055ee8e33df8a18c818c3026c9"752 },753 "tokens": {754 "bytes": 134698256,755 "path": "shards/shard-00020/tokens.bin",756 "sha256": "6995caf8a7e74ca7eb3e5f758cab5e2e8030e1e6d88f635757bef41f56f9204b"757 }758 },759 "partition": "train",760 "raw_path": "data/train-00020-of-00128.parquet",761 "shard_index": 20,762 "tokens": 33674564763 },764 {765 "document_ids_sha256": "d91370dc730d6f58e048dabce7022fd0035dc41cb409d7a220f56db6057865d5",766 "documents": 38262,767 "files": {768 "index": {769 "bytes": 3587856,770 "path": "shards/shard-00021/index.parquet",771 "sha256": "0d151d65f52bb90478d1e924a268346f3991746859d2afacc669bd81e5e88f74"772 },773 "metadata": {774 "bytes": 24039188,775 "path": "shards/shard-00021/metadata.parquet",776 "sha256": "f60a015c61196a8d48b9f3b58650aa4e126846adebf44af8920c22b44ab10b90"777 },778 "offsets": {779 "bytes": 306104,780 "path": "shards/shard-00021/offsets.bin",781 "sha256": "71b6090d40e9d3aefefb38b28945f6fa83afa59136f38a27ffc7e65b13d3a2e8"782 },783 "tokens": {784 "bytes": 131984660,785 "path": "shards/shard-00021/tokens.bin",786 "sha256": "7b26b4b2c7328d53fbddfd2b054b923d210398108b9545d58af812feb5439e19"787 }788 },789 "partition": "train",790 "raw_path": "data/train-00021-of-00128.parquet",791 "shard_index": 21,792 "tokens": 32996165793 },794 {795 "document_ids_sha256": "68c84fdc935d273da31c42b1c18e0b8743abe3e75f9a4d4720391e28e52ef9af",796 "documents": 38272,797 "files": {798 "index": {799 "bytes": 3587913,800 "path": "shards/shard-00022/index.parquet",801 "sha256": "d0758210988ed807e82e060260336a6ab7fd0bb9a8ada8be2c9ba60c0c2bc8cf"802 },803 "metadata": {804 "bytes": 24142083,805 "path": "shards/shard-00022/metadata.parquet",806 "sha256": "49f78589d72fda6810c7ef0b9c47cc4e51145344bd6adb9d9418aa25a71485cd"807 },808 "offsets": {809 "bytes": 306184,810 "path": "shards/shard-00022/offsets.bin",811 "sha256": "7ade1d40c8b45a44874c2fc5267e6791f592475bad6414f8321e54ebeddceef1"812 },813 "tokens": {814 "bytes": 133228852,815 "path": "shards/shard-00022/tokens.bin",816 "sha256": "49a5bf721a692e61b65f73910eb15cd85d87b0fb072d38cac29d21edb6ee0cab"817 }818 },819 "partition": "train",820 "raw_path": "data/train-00022-of-00128.parquet",821 "shard_index": 22,822 "tokens": 33307213823 },824 {825 "document_ids_sha256": "a561e4a296fd9c8f87c777def6fe7cb25788b25e6c0bc7d07f0776c92f3aed75",826 "documents": 38304,827 "files": {828 "index": {829 "bytes": 3591081,830 "path": "shards/shard-00023/index.parquet",831 "sha256": "0920fc9f23af632cd75d63c24f4d44a66435951f7bcd285a6d5e13ce35cb5cf0"832 },833 "metadata": {834 "bytes": 24239519,835 "path": "shards/shard-00023/metadata.parquet",836 "sha256": "027ebf3680067739a09d7587e625e0a1cbfacfe35c3f3ad32728aba38b2c8a95"837 },838 "offsets": {839 "bytes": 306440,840 "path": "shards/shard-00023/offsets.bin",841 "sha256": "c0016a9de0b0bd6c00f87f84fb2b84ade03ecd3973c6ef50587c35b14270e245"842 },843 "tokens": {844 "bytes": 133191304,845 "path": "shards/shard-00023/tokens.bin",846 "sha256": "46c1a8d9e0a70fa2f2981398377b788b0ae6caa61159a0c7e9a0dce76abaf6e4"847 }848 },849 "partition": "train",850 "raw_path": "data/train-00023-of-00128.parquet",851 "shard_index": 23,852 "tokens": 33297826853 },854 {855 "document_ids_sha256": "fb564a52d0b86d3eca17be50848c8957e93ecba426a711628d40f311dee7c2d4",856 "documents": 38207,857 "files": {858 "index": {859 "bytes": 3584051,860 "path": "shards/shard-00024/index.parquet",861 "sha256": "97ae493b466d60cd4cd166d3afa5bad3b9a09d1f94177555afe677d51540ba45"862 },863 "metadata": {864 "bytes": 24074956,865 "path": "shards/shard-00024/metadata.parquet",866 "sha256": "59364756fb837195406a4164d4675436abc490eb980a9f27e80fb63dc3929779"867 },868 "offsets": {869 "bytes": 305664,870 "path": "shards/shard-00024/offsets.bin",871 "sha256": "37ffc83e292290fb377edd2e6406fee98e3bf912fbfdbfa596e83b6838818fa9"872 },873 "tokens": {874 "bytes": 133428520,875 "path": "shards/shard-00024/tokens.bin",876 "sha256": "e2cffad0291b8a39ecae5942d1094afc89d0da3b16525bcb57d8102ef30fae14"877 }878 },879 "partition": "train",880 "raw_path": "data/train-00024-of-00128.parquet",881 "shard_index": 24,882 "tokens": 33357130883 },884 {885 "document_ids_sha256": "59401049e99ba5ca0fe11748c58529e2820c485db2042e7e3f374fb4a4c5d6a7",886 "documents": 38358,887 "files": {888 "index": {889 "bytes": 3599669,890 "path": "shards/shard-00025/index.parquet",891 "sha256": "b08ed9e33b731a7bea7e5d86755538174e84ee1bb4234088b05312155a8bd3ce"892 },893 "metadata": {894 "bytes": 24139203,895 "path": "shards/shard-00025/metadata.parquet",896 "sha256": "aca3b1d39df7beb6cb5b30e6755893cd64034ba9933ef13285d4e51acc6f8c6a"897 },898 "offsets": {899 "bytes": 306872,900 "path": "shards/shard-00025/offsets.bin",901 "sha256": "4aa89ddd89ad2372987699b6c35560f700c1e7f90eb6a350fba044f33cea3720"902 },903 "tokens": {904 "bytes": 133462360,905 "path": "shards/shard-00025/tokens.bin",906 "sha256": "92134d769a4cb46e5747e27fde25605e947ff5108730c1c183f8f32141aa432c"907 }908 },909 "partition": "train",910 "raw_path": "data/train-00025-of-00128.parquet",911 "shard_index": 25,912 "tokens": 33365590913 },914 {915 "document_ids_sha256": "38fc9001f9d6a66fdaf77221881e7e1edf53249ba2dea459f7d1efbfccc6372b",916 "documents": 38342,917 "files": {918 "index": {919 "bytes": 3595569,920 "path": "shards/shard-00026/index.parquet",921 "sha256": "d5118a758c31e7200450db4b85c8de3bac0da3acf2f20ea9f7dceb804e58360c"922 },923 "metadata": {924 "bytes": 24164196,925 "path": "shards/shard-00026/metadata.parquet",926 "sha256": "0cecdeafb72d6b677b63edadb8caee68d1ef16f96067b5ec03687e3f731f44dd"927 },928 "offsets": {929 "bytes": 306744,930 "path": "shards/shard-00026/offsets.bin",931 "sha256": "f4d64b32b14d1587dac2cfd95c659910a749b468f9acaaafead7c813ab657006"932 },933 "tokens": {934 "bytes": 133958232,935 "path": "shards/shard-00026/tokens.bin",936 "sha256": "0e066a1d3e80c30037adc5028a20949a3bd2589e1c4748a1b4a9552b682db320"937 }938 },939 "partition": "train",940 "raw_path": "data/train-00026-of-00128.parquet",941 "shard_index": 26,942 "tokens": 33489558943 },944 {945 "document_ids_sha256": "61ab7e7e91422a487d017279ce14a4e69e18f603926e124c56a516ca3591c8cd",946 "documents": 38508,947 "files": {948 "index": {949 "bytes": 3609210,950 "path": "shards/shard-00027/index.parquet",951 "sha256": "245ead6909aa8e6e800f7effe2569ae353d0a7eeafb8537ca61219355cdaa7f7"952 },953 "metadata": {954 "bytes": 24391327,955 "path": "shards/shard-00027/metadata.parquet",956 "sha256": "006b53557c739d0c8c7e230c174f1d88eb14b862f5dfca12bebd5f73070e02cb"957 },958 "offsets": {959 "bytes": 308072,960 "path": "shards/shard-00027/offsets.bin",961 "sha256": "996096dabb08130166cd21fa19dcad1ff9ea6fb7a6d59c699413b03c66ba97d6"962 },963 "tokens": {964 "bytes": 135096572,965 "path": "shards/shard-00027/tokens.bin",966 "sha256": "ea5f27f48629256c64bb83b11c45c0afd08f5939686f55d0d1fefaf29305609c"967 }968 },969 "partition": "train",970 "raw_path": "data/train-00027-of-00128.parquet",971 "shard_index": 27,972 "tokens": 33774143973 },974 {975 "document_ids_sha256": "0f45c112d6ffa63fe91e3536cfde1a0c42ff8d855e8c40157836a9eb53205f67",976 "documents": 38246,977 "files": {978 "index": {979 "bytes": 3586497,980 "path": "shards/shard-00028/index.parquet",981 "sha256": "f6b4463c3b44e026d648c04b059e12879c98a50d963b0fe121fdb2b0f92819a1"982 },983 "metadata": {984 "bytes": 24287092,985 "path": "shards/shard-00028/metadata.parquet",986 "sha256": "62e5917345f4c424f1a74fe5d14b1ed21aa9d0beac40d93fd301e6f2a9d967bb"987 },988 "offsets": {989 "bytes": 305976,990 "path": "shards/shard-00028/offsets.bin",991 "sha256": "8d4a264d7e8279e5f03d933099c296c40b4762df986ad62d72188402721f1438"992 },993 "tokens": {994 "bytes": 133960404,995 "path": "shards/shard-00028/tokens.bin",996 "sha256": "1281bc377596bb10ffb8c583a4db69233f356731c71cf0bfd16b92f3816cb5e8"997 }998 },999 "partition": "train",1000 "raw_path": "data/train-00028-of-00128.parquet",1001 "shard_index": 28,1002 "tokens": 334901011003 },1004 {1005 "document_ids_sha256": "551b11af64ce8faf297a7f904a51c10232f810f5065136bf4f88019ab5a64311",1006 "documents": 38307,1007 "files": {1008 "index": {1009 "bytes": 3592188,1010 "path": "shards/shard-00029/index.parquet",1011 "sha256": "d9486deb0244d0c7fd244024209544b49cd2dedaa5879c124a0fd5ef4a9c5fd9"1012 },1013 "metadata": {1014 "bytes": 24119557,1015 "path": "shards/shard-00029/metadata.parquet",1016 "sha256": "41dd3c4d7f1036968f2eb46f6df93f9e095f7df8f43ea2b0bdf11164d11e281e"1017 },1018 "offsets": {1019 "bytes": 306464,1020 "path": "shards/shard-00029/offsets.bin",1021 "sha256": "4cb04a245730de2195d08801b5ff020919b1de7ed6ddf8baaf6f7bc7e0aa6336"1022 },1023 "tokens": {1024 "bytes": 133333292,1025 "path": "shards/shard-00029/tokens.bin",1026 "sha256": "9551c577807aac8c7930a30405d52490b5166c195ef7678ba6835c9a8249159f"1027 }1028 },1029 "partition": "train",1030 "raw_path": "data/train-00029-of-00128.parquet",1031 "shard_index": 29,1032 "tokens": 333333231033 },1034 {1035 "document_ids_sha256": "c33c2faa170ce74ce2e9fdd903cfddcc42cbde0e63e56605b06a5aa551541d54",1036 "documents": 38264,1037 "files": {1038 "index": {1039 "bytes": 3584990,1040 "path": "shards/shard-00030/index.parquet",1041 "sha256": "588167e0388b853948fc731b4665df0a5048eaf95fde31bd62cd9f2b1aa05d9d"1042 },1043 "metadata": {1044 "bytes": 24288600,1045 "path": "shards/shard-00030/metadata.parquet",1046 "sha256": "d9d2edd78dd78ddf2bc2c1e60a349257c0ffe9c79a1090e2ae03863d64b4b936"1047 },1048 "offsets": {1049 "bytes": 306120,1050 "path": "shards/shard-00030/offsets.bin",1051 "sha256": "290683d54764136cf7146ee9b0329851637d791a7e468d94e335ac7412581aca"1052 },1053 "tokens": {1054 "bytes": 133891152,1055 "path": "shards/shard-00030/tokens.bin",1056 "sha256": "69f6edc105207a64ab97fc695e236bcf7f0869159736d233ea65f884d46a1ab9"1057 }1058 },1059 "partition": "train",1060 "raw_path": "data/train-00030-of-00128.parquet",1061 "shard_index": 30,1062 "tokens": 334727881063 },1064 {1065 "document_ids_sha256": "d8ec408d0bca55fb055b041bfc8c608df797f8a38225e16fd75f20b75f27161f",1066 "documents": 37718,1067 "files": {1068 "index": {1069 "bytes": 3536009,1070 "path": "shards/shard-00031/index.parquet",1071 "sha256": "171b1a0ede14436a89e5bdac995948a7828d53f222475784de6dab8a77e4e85d"1072 },1073 "metadata": {1074 "bytes": 23759409,1075 "path": "shards/shard-00031/metadata.parquet",1076 "sha256": "5c1c50d6ae7216a7b97addd596e2b0f760b44c30a0be64ce10370e80eecf4cbe"1077 },1078 "offsets": {1079 "bytes": 301752,1080 "path": "shards/shard-00031/offsets.bin",1081 "sha256": "ec7b2cfe79c86807f3023977996104417e55aff5e7e161d9e29fa32c433ecb80"1082 },1083 "tokens": {1084 "bytes": 132059776,1085 "path": "shards/shard-00031/tokens.bin",1086 "sha256": "f3f898441d021fcac8fdbb5aa08585ddb19e292c04804256d16c1ce37749831a"1087 }1088 },1089 "partition": "train",1090 "raw_path": "data/train-00031-of-00128.parquet",1091 "shard_index": 31,1092 "tokens": 330149441093 },1094 {1095 "document_ids_sha256": "7283fca0578753a22210c263cffd8646a2ee6221a706fd46ebdc92d6c4d8b82b",1096 "documents": 38421,1097 "files": {1098 "index": {1099 "bytes": 3604621,1100 "path": "shards/shard-00032/index.parquet",1101 "sha256": "4b3364672599c232181538543666cea6e0df3208cffcc4103b834468ae78ba57"1102 },1103 "metadata": {1104 "bytes": 24215407,1105 "path": "shards/shard-00032/metadata.parquet",1106 "sha256": "7041106dd74bbb6dc1c58424e0166ffd72c538ae0c1786ebeb938659bbbae232"1107 },1108 "offsets": {1109 "bytes": 307376,1110 "path": "shards/shard-00032/offsets.bin",1111 "sha256": "3f3840824d82a5b0fe86e261037310d5f5f9fa7921e500fefe2daa969b8f1a14"1112 },1113 "tokens": {1114 "bytes": 133988160,1115 "path": "shards/shard-00032/tokens.bin",1116 "sha256": "df9c58014133d1ff7f13aafcdbeada930be1cdd613ba43d8044a0be55a6f0082"1117 }1118 },1119 "partition": "train",1120 "raw_path": "data/train-00032-of-00128.parquet",1121 "shard_index": 32,1122 "tokens": 334970401123 },1124 {1125 "document_ids_sha256": "b2b13e7ab9f7f8b8d2270fbc085d8be130fb5894cd3ee5ffcdbdd02cb6cf95c6",1126 "documents": 38278,1127 "files": {1128 "index": {1129 "bytes": 3592537,1130 "path": "shards/shard-00033/index.parquet",1131 "sha256": "cce5033690fdf5c526be84233d787251f1169913ae26c8a6f84f1d6a3eb2940e"1132 },1133 "metadata": {1134 "bytes": 24218500,1135 "path": "shards/shard-00033/metadata.parquet",1136 "sha256": "9f4ea7edc677283260847a187efe7416de779c5d86bf4d56cb52aec6f87da7aa"1137 },1138 "offsets": {1139 "bytes": 306232,1140 "path": "shards/shard-00033/offsets.bin",1141 "sha256": "0dbd5eef28e055164d6eb7c8a2b8bac032c6b73be412eb2e124a27f037530f84"1142 },1143 "tokens": {1144 "bytes": 134278580,1145 "path": "shards/shard-00033/tokens.bin",1146 "sha256": "e845de46bb6a14a91e1e06701987c2ad8ca8aec7bdbae3fb5a2df93ce012c1fe"1147 }1148 },1149 "partition": "train",1150 "raw_path": "data/train-00033-of-00128.parquet",1151 "shard_index": 33,1152 "tokens": 335696451153 },1154 {1155 "document_ids_sha256": "100a78bf2fe7bbdd53839cc06a602af536bb5933ac1a1a5ebada818fa057edd6",1156 "documents": 38142,1157 "files": {1158 "index": {1159 "bytes": 3573897,1160 "path": "shards/shard-00034/index.parquet",1161 "sha256": "fb7e101ab3a6890e09379905d9d0adac02b7c0829eede673d4c73a9ecd92c31a"1162 },1163 "metadata": {1164 "bytes": 24082791,1165 "path": "shards/shard-00034/metadata.parquet",1166 "sha256": "ce7a6cf170bc125db1afe7a604f0ba0a3fbb6cdc3c08242ad5fca38b25acdbe7"1167 },1168 "offsets": {1169 "bytes": 305144,1170 "path": "shards/shard-00034/offsets.bin",1171 "sha256": "7b5afbfb78385c5419dd2127846348a35e0aa61a119ae3e8f27b87622e5407fb"1172 },1173 "tokens": {1174 "bytes": 132795680,1175 "path": "shards/shard-00034/tokens.bin",1176 "sha256": "7e6820b40df5fae9db69f3a48f930fbe156183986a81da3d2ea12b2bf2fceddf"1177 }1178 },1179 "partition": "train",1180 "raw_path": "data/train-00034-of-00128.parquet",1181 "shard_index": 34,1182 "tokens": 331989201183 },1184 {1185 "document_ids_sha256": "e4fb17b01f7f8f83f3e0baa5ff7f5fdd6f530fb00cb836819d00a9647df82fbc",1186 "documents": 37886,1187 "files": {1188 "index": {1189 "bytes": 3551623,1190 "path": "shards/shard-00035/index.parquet",1191 "sha256": "966cf5db20cca2f82e76957d665dfe438f11f1df89c84a2b75e016b4e8bfbcd3"1192 },1193 "metadata": {1194 "bytes": 23888923,1195 "path": "shards/shard-00035/metadata.parquet",1196 "sha256": "a682dabf28415ffa48e8f668833314e2b2e0fe1d91d863d9a57468f55901a8fd"1197 },1198 "offsets": {1199 "bytes": 303096,1200 "path": "shards/shard-00035/offsets.bin",