shau1905/python_3

1{2  "add_prefix_space": null,3  "backend": "tokenizers",4  "bos_token": "<s>",5  "clean_up_tokenization_spaces": false,6  "eos_token": "<|endoftext|>",7  "is_local": false,8  "legacy": false,9  "model_max_length": 4096,10  "pad_token": "<|placeholder6|>",11  "padding_side": "left",12  "sp_model_kwargs": {},13  "tokenizer_class": "TokenizersBackend",14  "unk_token": "<unk>",15  "use_default_system_prompt": false,16  "added_tokens_decoder": {17    "0": {18      "content": "<unk>",19      "single_word": false,20      "lstrip": false,21      "rstrip": false,22      "normalized": false,23      "special": true24    },25    "1": {26      "content": "<s>",27      "single_word": false,28      "lstrip": false,29      "rstrip": false,30      "normalized": false,31      "special": true32    },33    "2": {34      "content": "</s>",35      "single_word": false,36      "lstrip": false,37      "rstrip": true,38      "normalized": false,39      "special": false40    },41    "32000": {42      "content": "<|endoftext|>",43      "single_word": false,44      "lstrip": false,45      "rstrip": false,46      "normalized": false,47      "special": true48    },49    "32001": {50      "content": "<|assistant|>",51      "single_word": false,52      "lstrip": false,53      "rstrip": true,54      "normalized": false,55      "special": true56    },57    "32002": {58      "content": "<|placeholder1|>",59      "single_word": false,60      "lstrip": false,61      "rstrip": true,62      "normalized": false,63      "special": true64    },65    "32003": {66      "content": "<|placeholder2|>",67      "single_word": false,68      "lstrip": false,69      "rstrip": true,70      "normalized": false,71      "special": true72    },73    "32004": {74      "content": "<|placeholder3|>",75      "single_word": false,76      "lstrip": false,77      "rstrip": true,78      "normalized": false,79      "special": true80    },81    "32005": {82      "content": "<|placeholder4|>",83      "single_word": false,84      "lstrip": false,85      "rstrip": true,86      "normalized": false,87      "special": true88    },89    "32006": {90      "content": "<|system|>",91      "single_word": false,92      "lstrip": false,93      "rstrip": true,94      "normalized": false,95      "special": true96    },97    "32007": {98      "content": "<|end|>",99      "single_word": false,100      "lstrip": false,101      "rstrip": true,102      "normalized": false,103      "special": true104    },105    "32008": {106      "content": "<|placeholder5|>",107      "single_word": false,108      "lstrip": false,109      "rstrip": true,110      "normalized": false,111      "special": true112    },113    "32009": {114      "content": "<|placeholder6|>",115      "single_word": false,116      "lstrip": false,117      "rstrip": false,118      "normalized": false,119      "special": true120    },121    "32010": {122      "content": "<|user|>",123      "single_word": false,124      "lstrip": false,125      "rstrip": true,126      "normalized": false,127      "special": true128    }129  }130}131