RichardErkhov/bigcode_-_gpt_bigcode-santacoder-8bits

1{2  "add_prefix_space": false,3  "added_tokens_decoder": {4    "49152": {5      "content": "<|endoftext|>",6      "lstrip": false,7      "normalized": false,8      "rstrip": false,9      "single_word": false,10      "special": true11    },12    "49153": {13      "content": "<fim-prefix>",14      "lstrip": false,15      "normalized": false,16      "rstrip": false,17      "single_word": false,18      "special": true19    },20    "49154": {21      "content": "<fim-middle>",22      "lstrip": false,23      "normalized": false,24      "rstrip": false,25      "single_word": false,26      "special": true27    },28    "49155": {29      "content": "<fim-suffix>",30      "lstrip": false,31      "normalized": false,32      "rstrip": false,33      "single_word": false,34      "special": true35    },36    "49156": {37      "content": "<fim-pad>",38      "lstrip": false,39      "normalized": false,40      "rstrip": false,41      "single_word": false,42      "special": true43    }44  },45  "additional_special_tokens": [46    "<|endoftext|>",47    "<fim-prefix>",48    "<fim-middle>",49    "<fim-suffix>",50    "<fim-pad>"51  ],52  "bos_token": "<|endoftext|>",53  "clean_up_tokenization_spaces": true,54  "eos_token": "<|endoftext|>",55  "errors": "replace",56  "model_max_length": 2048,57  "tokenizer_class": "GPT2Tokenizer",58  "unk_token": "<|endoftext|>"59}60