Modelpublic

KRISHNAPURI/q-trust-codebert

sourceHugging Facemitupdated 21d agoView on Hugging Face

1likes112downloads

rootcrypto_codeberttokenizer_config.json

tokenizer_config.json19 linesDownload Raw Back to crypto_codebert

1{2  "add_prefix_space": false,3  "backend": "tokenizers",4  "bos_token": "<s>",5  "cls_token": "<s>",6  "eos_token": "</s>",7  "errors": "replace",8  "is_local": false,9  "local_files_only": false,10  "mask_token": "<mask>",11  "max_len": 512,12  "model_max_length": 512,13  "pad_token": "<pad>",14  "sep_token": "</s>",15  "tokenizer_class": "RobertaTokenizer",16  "trim_offsets": true,17  "unk_token": "<unk>"18}19