Team Ai
Modelpublic

optimum-intel-internal-testing/tiny-random-phi-4-multimodal

sourceHugging Faceupdated 1y agoView on Hugging Face
0likes16kdownloads
tokenizer_config.json127 linesDownload Raw Back to root
1{2  "add_prefix_space": false,3  "added_tokens_decoder": {4    "199999": {5      "content": "<|endoftext|>",6      "lstrip": false,7      "normalized": false,8      "rstrip": false,9      "single_word": false,10      "special": true11    },12    "200010": {13      "content": "<|endoftext10|>",14      "lstrip": false,15      "normalized": false,16      "rstrip": false,17      "single_word": false,18      "special": true19    },20    "200011": {21      "content": "<|endoftext11|>",22      "lstrip": false,23      "normalized": false,24      "rstrip": false,25      "single_word": false,26      "special": true27    },28    "200018": {29      "content": "<|endofprompt|>",30      "lstrip": false,31      "normalized": false,32      "rstrip": false,33      "single_word": false,34      "special": true35    },36    "200019": {37      "content": "<|assistant|>",38      "lstrip": false,39      "normalized": false,40      "rstrip": true,41      "single_word": false,42      "special": true43    },44    "200020": {45      "content": "<|end|>",46      "lstrip": false,47      "normalized": false,48      "rstrip": true,49      "single_word": false,50      "special": true51    },52    "200021": {53      "content": "<|user|>",54      "lstrip": false,55      "normalized": false,56      "rstrip": true,57      "single_word": false,58      "special": true59    },60    "200022": {61      "content": "<|system|>",62      "lstrip": false,63      "normalized": false,64      "rstrip": true,65      "single_word": false,66      "special": true67    },68    "200023": {69      "content": "<|tool|>",70      "lstrip": false,71      "normalized": false,72      "rstrip": true,73      "single_word": false,74      "special": false75    },76    "200024": {77      "content": "<|/tool|>",78      "lstrip": false,79      "normalized": false,80      "rstrip": true,81      "single_word": false,82      "special": false83    },84    "200025": {85      "content": "<|tool_call|>",86      "lstrip": false,87      "normalized": false,88      "rstrip": true,89      "single_word": false,90      "special": false91    },92    "200026": {93      "content": "<|/tool_call|>",94      "lstrip": false,95      "normalized": false,96      "rstrip": true,97      "single_word": false,98      "special": false99    },100    "200027": {101      "content": "<|tool_response|>",102      "lstrip": false,103      "normalized": false,104      "rstrip": true,105      "single_word": false,106      "special": false107    },108    "200028": {109      "content": "<|tag|>",110      "lstrip": false,111      "normalized": false,112      "rstrip": true,113      "single_word": false,114      "special": true115    }116  },117  "bos_token": "<|endoftext|>",118  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}",119  "clean_up_tokenization_spaces": false,120  "eos_token": "<|endoftext|>",121  "extra_special_tokens": {},122  "model_max_length": 131072,123  "pad_token": "<|endoftext|>",124  "tokenizer_class": "GPT2Tokenizer",125  "unk_token": "<|endoftext|>"126}127