Bochkov/ab_ext_binary16

1{2  "architectures": [3    "AttnExtForCausalLM"4  ],5  "attention_bias": false,6  "auto_map": {7    "AutoConfig": "configuration_attn_ext.AttnExtConfig",8    "AutoModelForCausalLM": "modeling_attn_ext.AttnExtForCausalLM"9  },10  "binary_dim": 16,11  "binary_encoding": "zero_one",12  "binary_repeat": 128,13  "binary_scale": 1.0,14  "block_size": 2048,15  "bos_token_id": 0,16  "code_seed": 12345,17  "d_model": 2048,18  "dropout": 0.0,19  "eos_token_id": 0,20  "ffn_multiplier": 4.0,21  "head_dim": 64,22  "hidden_size": 2048,23  "initializer_range": 0.02,24  "input_mode": "binary16",25  "is_decoder": true,26  "is_encoder_decoder": false,27  "max_position_embeddings": 2048,28  "min_col_weight": 4,29  "min_row_weight": 4,30  "mlp_bias": false,31  "model_type": "attn_ext",32  "multiple_of": 256,33  "n_head": 32,34  "n_layer": 24,35  "num_attention_heads": 32,36  "num_hidden_layers": 24,37  "pad_token_id": null,38  "rms_norm_eps": 1e-05,39  "rope_theta": 10000.0,40  "tie_word_embeddings": false,41  "transformers_version": null,42  "use_cache": false,43  "vocab_size": 4915244}45