trl-internal-testing/tiny-Qwen3_5ForConditionalGeneration-NoThink
099k
1{2 "architectures": [3 "Qwen3_5ForConditionalGeneration"4 ],5 "dtype": "bfloat16",6 "image_token_id": 248056,7 "model_type": "qwen3_5",8 "text_config": {9 "attention_bias": false,10 "attention_dropout": 0.0,11 "attn_output_gate": true,12 "bos_token_id": null,13 "dtype": "bfloat16",14 "eos_token_id": 248044,15 "full_attention_interval": 2,16 "head_dim": 256,17 "hidden_act": "silu",18 "hidden_size": 16,19 "initializer_range": 0.02,20 "intermediate_size": 3584,21 "layer_types": [22 "linear_attention",23 "full_attention"24 ],25 "linear_conv_kernel_dim": 4,26 "linear_key_head_dim": 128,27 "linear_num_key_heads": 16,28 "linear_num_value_heads": 16,29 "linear_value_head_dim": 128,30 "mamba_ssm_dtype": "float32",31 "max_position_embeddings": 262144,32 "mlp_only_layers": [],33 "model_type": "qwen3_5_text",34 "mtp_num_hidden_layers": 1,35 "mtp_use_dedicated_embeddings": false,36 "num_attention_heads": 4,37 "num_hidden_layers": 2,38 "num_key_value_heads": 2,39 "pad_token_id": null,40 "partial_rotary_factor": 0.25,41 "rms_norm_eps": 1e-06,42 "rope_parameters": {43 "mrope_interleaved": true,44 "mrope_section": [45 11,46 11,47 1048 ],49 "partial_rotary_factor": 0.25,50 "rope_theta": 10000000,51 "rope_type": "default"52 },53 "tie_word_embeddings": true,54 "use_cache": true,55 "vocab_size": 24832056 },57 "tie_word_embeddings": true,58 "transformers_version": "5.2.0",59 "video_token_id": 248057,60 "vision_config": {61 "deepstack_visual_indexes": [],62 "depth": 2,63 "hidden_act": "gelu_pytorch_tanh",64 "hidden_size": 16,65 "in_channels": 3,66 "initializer_range": 0.02,67 "intermediate_size": 32,68 "model_type": "qwen3_5",69 "num_heads": 4,70 "num_position_embeddings": 2304,71 "out_hidden_size": 16,72 "patch_size": 16,73 "spatial_merge_size": 2,74 "temporal_patch_size": 275 },76 "vision_end_token_id": 248054,77 "vision_start_token_id": 24805378}79