CoIR-Retrieval/CodeSearchNet-ccr

1---2dataset_info:3- config_name: corpus4  features:5  - name: _id6    dtype: string7  - name: title8    dtype: string9  - name: partition10    dtype: string11  - name: text12    dtype: string13  - name: language14    dtype: string15  - name: meta_information16    struct:17    - name: resource18      dtype: string19  splits:20  - name: corpus21    num_bytes: 607484622    num_examples: 2758823  download_size: 260957624  dataset_size: 607484625- config_name: go-corpus26  features:27  - name: _id28    dtype: string29  - name: title30    dtype: string31  - name: partition32    dtype: string33  - name: text34    dtype: string35  - name: language36    dtype: string37  - name: meta_information38    struct:39    - name: resource40      dtype: string41  splits:42  - name: corpus43    num_bytes: 3831030744    num_examples: 18273545  download_size: 1642193946  dataset_size: 3831030747- config_name: go-qrels48  features:49  - name: query-id50    dtype: string51  - name: corpus-id52    dtype: string53  - name: score54    dtype: int6455  splits:56  - name: train57    num_bytes: 479642058    num_examples: 16728859  - name: valid60    num_bytes: 21975061    num_examples: 732562  - name: test63    num_bytes: 24366064    num_examples: 812265  download_size: 207973866  dataset_size: 525983067- config_name: go-queries68  features:69  - name: _id70    dtype: string71  - name: title72    dtype: string73  - name: partition74    dtype: string75  - name: text76    dtype: string77  - name: language78    dtype: string79  - name: meta_information80    struct:81    - name: resource82      dtype: string83  splits:84  - name: queries85    num_bytes: 5248712786    num_examples: 18273587  download_size: 2251851188  dataset_size: 5248712789- config_name: java-corpus90  features:91  - name: _id92    dtype: string93  - name: title94    dtype: string95  - name: partition96    dtype: string97  - name: text98    dtype: string99  - name: language100    dtype: string101  - name: meta_information102    struct:103    - name: resource104      dtype: string105  splits:106  - name: corpus107    num_bytes: 52844259108    num_examples: 181061109  download_size: 20139955110  dataset_size: 52844259111- config_name: java-qrels112  features:113  - name: query-id114    dtype: string115  - name: corpus-id116    dtype: string117  - name: score118    dtype: int64119  splits:120  - name: train121    num_bytes: 4725470122    num_examples: 164923123  - name: valid124    num_bytes: 155490125    num_examples: 5183126  - name: test127    num_bytes: 328650128    num_examples: 10955129  download_size: 2059850130  dataset_size: 5209610131- config_name: java-queries132  features:133  - name: _id134    dtype: string135  - name: title136    dtype: string137  - name: partition138    dtype: string139  - name: text140    dtype: string141  - name: language142    dtype: string143  - name: meta_information144    struct:145    - name: resource146      dtype: string147  splits:148  - name: queries149    num_bytes: 76773974150    num_examples: 181061151  download_size: 29677644152  dataset_size: 76773974153- config_name: javascript-corpus154  features:155  - name: _id156    dtype: string157  - name: title158    dtype: string159  - name: partition160    dtype: string161  - name: text162    dtype: string163  - name: language164    dtype: string165  - name: meta_information166    struct:167    - name: resource168      dtype: string169  splits:170  - name: corpus171    num_bytes: 20940791172    num_examples: 65201173  download_size: 9148280174  dataset_size: 20940791175- config_name: javascript-qrels176  features:177  - name: query-id178    dtype: string179  - name: corpus-id180    dtype: string181  - name: score182    dtype: int64183  splits:184  - name: train185    num_bytes: 1602480186    num_examples: 58025187  - name: valid188    num_bytes: 108780189    num_examples: 3885190  - name: test191    num_bytes: 92148192    num_examples: 3291193  download_size: 743674194  dataset_size: 1803408195- config_name: javascript-queries196  features:197  - name: _id198    dtype: string199  - name: title200    dtype: string201  - name: partition202    dtype: string203  - name: text204    dtype: string205  - name: language206    dtype: string207  - name: meta_information208    struct:209    - name: resource210      dtype: string211  splits:212  - name: queries213    num_bytes: 28240438214    num_examples: 65201215  download_size: 12144058216  dataset_size: 28240438217- config_name: php-corpus218  features:219  - name: _id220    dtype: string221  - name: title222    dtype: string223  - name: partition224    dtype: string225  - name: text226    dtype: string227  - name: language228    dtype: string229  - name: meta_information230    struct:231    - name: resource232      dtype: string233  splits:234  - name: corpus235    num_bytes: 75580225236    num_examples: 268237237  download_size: 28610118238  dataset_size: 75580225239- config_name: php-qrels240  features:241  - name: query-id242    dtype: string243  - name: corpus-id244    dtype: string245  - name: score246    dtype: int64247  splits:248  - name: train249    num_bytes: 7015010250    num_examples: 241241251  - name: valid252    num_bytes: 389460253    num_examples: 12982254  - name: test255    num_bytes: 420420256    num_examples: 14014257  download_size: 3053262258  dataset_size: 7824890259- config_name: php-queries260  features:261  - name: _id262    dtype: string263  - name: title264    dtype: string265  - name: partition266    dtype: string267  - name: text268    dtype: string269  - name: language270    dtype: string271  - name: meta_information272    struct:273    - name: resource274      dtype: string275  splits:276  - name: queries277    num_bytes: 108819401278    num_examples: 268237279  download_size: 41855424280  dataset_size: 108819401281- config_name: python-corpus282  features:283  - name: _id284    dtype: string285  - name: title286    dtype: string287  - name: partition288    dtype: string289  - name: text290    dtype: string291  - name: language292    dtype: string293  - name: meta_information294    struct:295    - name: resource296      dtype: string297  splits:298  - name: corpus299    num_bytes: 114964501300    num_examples: 280652301  download_size: 45683292302  dataset_size: 114964501303- config_name: python-qrels304  features:305  - name: query-id306    dtype: string307  - name: corpus-id308    dtype: string309  - name: score310    dtype: int64311  splits:312  - name: train313    num_bytes: 7332380314    num_examples: 251820315  - name: valid316    num_bytes: 417420317    num_examples: 13914318  - name: test319    num_bytes: 447540320    num_examples: 14918321  download_size: 3193423322  dataset_size: 8197340323- config_name: python-queries324  features:325  - name: _id326    dtype: string327  - name: title328    dtype: string329  - name: partition330    dtype: string331  - name: text332    dtype: string333  - name: language334    dtype: string335  - name: meta_information336    struct:337    - name: resource338      dtype: string339  splits:340  - name: queries341    num_bytes: 164228951342    num_examples: 280652343  download_size: 68811755344  dataset_size: 164228951345- config_name: ruby-corpus346  features:347  - name: _id348    dtype: string349  - name: title350    dtype: string351  - name: partition352    dtype: string353  - name: text354    dtype: string355  - name: language356    dtype: string357  - name: meta_information358    struct:359    - name: resource360      dtype: string361  splits:362  - name: corpus363    num_bytes: 6074846364    num_examples: 27588365  download_size: 2609576366  dataset_size: 6074846367- config_name: ruby-qrels368  features:369  - name: query-id370    dtype: string371  - name: corpus-id372    dtype: string373  - name: score374    dtype: int64375  splits:376  - name: train377    num_bytes: 675736378    num_examples: 24927379  - name: valid380    num_bytes: 39200381    num_examples: 1400382  - name: test383    num_bytes: 35308384    num_examples: 1261385  download_size: 316849386  dataset_size: 750244387- config_name: ruby-queries388  features:389  - name: _id390    dtype: string391  - name: title392    dtype: string393  - name: partition394    dtype: string395  - name: text396    dtype: string397  - name: language398    dtype: string399  - name: meta_information400    struct:401    - name: resource402      dtype: string403  splits:404  - name: queries405    num_bytes: 8891575406    num_examples: 27588407  download_size: 3948111408  dataset_size: 8891575409configs:410- config_name: corpus411  data_files:412  - split: corpus413    path: corpus/corpus-*414- config_name: go-corpus415  data_files:416  - split: corpus417    path: go-corpus/corpus-*418- config_name: go-qrels419  data_files:420  - split: train421    path: go-qrels/train-*422  - split: valid423    path: go-qrels/valid-*424  - split: test425    path: go-qrels/test-*426- config_name: go-queries427  data_files:428  - split: queries429    path: go-queries/queries-*430- config_name: java-corpus431  data_files:432  - split: corpus433    path: java-corpus/corpus-*434- config_name: java-qrels435  data_files:436  - split: train437    path: java-qrels/train-*438  - split: valid439    path: java-qrels/valid-*440  - split: test441    path: java-qrels/test-*442- config_name: java-queries443  data_files:444  - split: queries445    path: java-queries/queries-*446- config_name: javascript-corpus447  data_files:448  - split: corpus449    path: javascript-corpus/corpus-*450- config_name: javascript-qrels451  data_files:452  - split: train453    path: javascript-qrels/train-*454  - split: valid455    path: javascript-qrels/valid-*456  - split: test457    path: javascript-qrels/test-*458- config_name: javascript-queries459  data_files:460  - split: queries461    path: javascript-queries/queries-*462- config_name: php-corpus463  data_files:464  - split: corpus465    path: php-corpus/corpus-*466- config_name: php-qrels467  data_files:468  - split: train469    path: php-qrels/train-*470  - split: valid471    path: php-qrels/valid-*472  - split: test473    path: php-qrels/test-*474- config_name: php-queries475  data_files:476  - split: queries477    path: php-queries/queries-*478- config_name: python-corpus479  data_files:480  - split: corpus481    path: python-corpus/corpus-*482- config_name: python-qrels483  data_files:484  - split: train485    path: python-qrels/train-*486  - split: valid487    path: python-qrels/valid-*488  - split: test489    path: python-qrels/test-*490- config_name: python-queries491  data_files:492  - split: queries493    path: python-queries/queries-*494- config_name: ruby-corpus495  data_files:496  - split: corpus497    path: ruby-corpus/corpus-*498- config_name: ruby-qrels499  data_files:500  - split: train501    path: ruby-qrels/train-*502  - split: valid503    path: ruby-qrels/valid-*504  - split: test505    path: ruby-qrels/test-*506- config_name: ruby-queries507  data_files:508  - split: queries509    path: ruby-queries/queries-*510---511Employing the MTEB evaluation framework's dataset version, utilize the code below for assessment:512 513```python514import mteb515import logging516from sentence_transformers import SentenceTransformer517from mteb import MTEB518 519logger = logging.getLogger(__name__)520 521model_name = 'intfloat/e5-base-v2'522model = SentenceTransformer(model_name)523tasks = mteb.get_tasks(524    tasks=[525        "AppsRetrieval",526        "CodeFeedbackMT",527        "CodeFeedbackST",528        "CodeTransOceanContest",529        "CodeTransOceanDL",530        "CosQA",531        "SyntheticText2SQL",532        "StackOverflowQA",533        "COIRCodeSearchNetRetrieval",534        "CodeSearchNetCCRetrieval",535    ]536)537evaluation = MTEB(tasks=tasks)538results = evaluation.run(539    model=model,540    overwrite_results=True541)542print(result)543```544Employing the MTEB evaluation framework's dataset version, utilize the code below for assessment:545 546```python547import mteb548import logging549from sentence_transformers import SentenceTransformer550from mteb import MTEB551 552logger = logging.getLogger(__name__)553 554model_name = 'intfloat/e5-base-v2'555model = SentenceTransformer(model_name)556tasks = mteb.get_tasks(557    tasks=[558        "AppsRetrieval",559        "CodeFeedbackMT",560        "CodeFeedbackST",561        "CodeTransOceanContest",562        "CodeTransOceanDL",563        "CosQA",564        "SyntheticText2SQL",565        "StackOverflowQA",566        "COIRCodeSearchNetRetrieval",567        "CodeSearchNetCCRetrieval",568    ]569)570evaluation = MTEB(tasks=tasks)571results = evaluation.run(572    model=model,573    overwrite_results=True574)575print(result)576```