ObscuraCoder/commit-chronicle

1---2dataset_info:3- config_name: C4  features:5  - name: message6    dtype: string7  - name: diff8    dtype: string9  splits:10  - name: train11    num_bytes: 11601089112    num_examples: 13890313  - name: validation14    num_bytes: 2365514715    num_examples: 2873916  - name: test17    num_bytes: 1508957318    num_examples: 1810819  download_size: 8421824520  dataset_size: 15475561121- config_name: C++22  features:23  - name: message24    dtype: string25  - name: diff26    dtype: string27  splits:28  - name: train29    num_bytes: 28398820330    num_examples: 31885831  - name: validation32    num_bytes: 7134748033    num_examples: 8187534  - name: test35    num_bytes: 4043452736    num_examples: 4596537  download_size: 21005478938  dataset_size: 39577021039- config_name: Go40  features:41  - name: message42    dtype: string43  - name: diff44    dtype: string45  splits:46  - name: train47    num_bytes: 23334877848    num_examples: 26623649  - name: validation50    num_bytes: 5091690551    num_examples: 5595952  - name: test53    num_bytes: 4901831454    num_examples: 5527855  download_size: 17730004056  dataset_size: 33328399757- config_name: Java58  features:59  - name: message60    dtype: string61  - name: diff62    dtype: string63  splits:64  - name: train65    num_bytes: 34109598966    num_examples: 35211867  - name: validation68    num_bytes: 7014285369    num_examples: 7517370  - name: test71    num_bytes: 7374646272    num_examples: 7578973  download_size: 23771015874  dataset_size: 48498530475- config_name: Python76  features:77  - name: message78    dtype: string79  - name: diff80    dtype: string81  splits:82  - name: train83    num_bytes: 54024975284    num_examples: 62700785  - name: validation86    num_bytes: 8352163687    num_examples: 9814788  - name: test89    num_bytes: 9283680590    num_examples: 11351791  download_size: 39061438592  dataset_size: 71660819393- config_name: Rust94  features:95  - name: message96    dtype: string97  - name: diff98    dtype: string99  splits:100  - name: train101    num_bytes: 78635879102    num_examples: 88593103  - name: validation104    num_bytes: 20637229105    num_examples: 23356106  - name: test107    num_bytes: 15019809108    num_examples: 16851109  download_size: 61646506110  dataset_size: 114292917111- config_name: TypeScript112  features:113  - name: message114    dtype: string115  - name: diff116    dtype: string117  splits:118  - name: train119    num_bytes: 306561530120    num_examples: 378871121  - name: validation122    num_bytes: 56861379123    num_examples: 72008124  - name: test125    num_bytes: 62396070126    num_examples: 77097127  download_size: 224008717128  dataset_size: 425818979129configs:130- config_name: C131  data_files:132  - split: train133    path: C/train-*134  - split: validation135    path: C/validation-*136  - split: test137    path: C/test-*138- config_name: C++139  data_files:140  - split: train141    path: C++/train-*142  - split: validation143    path: C++/validation-*144  - split: test145    path: C++/test-*146- config_name: Go147  data_files:148  - split: train149    path: Go/train-*150  - split: validation151    path: Go/validation-*152  - split: test153    path: Go/test-*154- config_name: Java155  data_files:156  - split: train157    path: Java/train-*158  - split: validation159    path: Java/validation-*160  - split: test161    path: Java/test-*162- config_name: Python163  data_files:164  - split: train165    path: Python/train-*166  - split: validation167    path: Python/validation-*168  - split: test169    path: Python/test-*170- config_name: Rust171  data_files:172  - split: train173    path: Rust/train-*174  - split: validation175    path: Rust/validation-*176  - split: test177    path: Rust/test-*178- config_name: TypeScript179  data_files:180  - split: train181    path: TypeScript/train-*182  - split: validation183    path: TypeScript/validation-*184  - split: test185    path: TypeScript/test-*186tags:187- code188size_categories:189- 100K<n<1M190---191 192This is a filtered version of the [`JetBrains-Research/commit-chronicle`](https://huggingface.co/datasets/JetBrains-Research/commit-chronicle) dataset. It has been subsetted for the following languages:193 194```python195[196  "C", "C++", "Go", "Java", "Python", "Rust", "TypeScript"197]198```199 200Further filtering steps undertaken are:201 2021. Useless features have been removed and only the message and diff retained.2032. Only commits that modify a single file have been chosen.2043. Samples containing diffs longer than 1024 tokens (by the [`ObscuraCoder/Tokenizer`](https://huggingface.co/ObscuraCoder/Tokenizer) tokenizer estimate) have been discarded.