microsoft/longcoder-base

1{2  "architectures": [3    "LongformerModel"4  ],5  "attention_probs_dropout_prob": 0.1,6  "attention_window": [7    1024,8    1024,9    1024,10    1024,11    1024,12    1024,13    1024,14    1024,15    1024,16    1024,17    1024,18    102419  ],20  "bos_token_id": 0,21  "classifier_dropout": null,22  "eos_token_id": 2,23  "gradient_checkpointing": false,24  "hidden_act": "gelu",25  "hidden_dropout_prob": 0.1,26  "hidden_size": 768,27  "initializer_range": 0.02,28  "intermediate_size": 3072,29  "is_decoder_only": true,30  "layer_norm_eps": 1e-05,31  "max_position_embeddings": 4098,32  "model_type": "longformer",33  "num_attention_heads": 12,34  "num_hidden_layers": 12,35  "onnx_export": false,36  "output_past": true,37  "pad_token_id": 1,38  "position_embedding_type": "absolute",39  "sep_token_id": 2,40  "torch_dtype": "float32",41  "transformers_version": "4.25.0.dev0",42  "type_vocab_size": 10,43  "use_cache": true,44  "vocab_size": 5141645}46