explosion-testing/refined-web-model-new-decoder-test

1{2  "alibi": false,3  "apply_residual_connection_post_layernorm": false,4  "architectures": [5    "RWModel"6  ],7  "attention_dropout": 0.0,8  "auto_map": {9    "AutoConfig": "configuration_RW.RWConfig",10    "AutoModel": "modelling_RW.RWModel",11    "AutoModelForCausalLM": "modelling_RW.RWForCausalLM",12    "AutoModelForQuestionAnswering": "modelling_RW.RWForQuestionAnswering",13    "AutoModelForSequenceClassification": "modelling_RW.RWForSequenceClassification",14    "AutoModelForTokenClassification": "modelling_RW.RWForTokenClassification"15  },16  "bias": false,17  "bos_token_id": 11,18  "eos_token_id": 11,19  "hidden_dropout": 0.0,20  "hidden_size": 256,21  "initializer_range": 0.02,22  "layer_norm_epsilon": 1e-05,23  "model_type": "RefinedWeb",24  "n_head": 4,25  "n_head_kv": 2,26  "n_layer": 5,27  "parallel_attn": true,28  "torch_dtype": "float32",29  "transformers_version": "4.30.2",30  "use_cache": true,31  "vocab_size": 102432}33