NovelAI/genji-python-6B

1{2    "activation_function": "gelu_new",3    "architectures": [4      "GPTNeoForCausalLM"5    ],6    "attention_dropout": 0,7    "attention_layers": ["global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global", "global"],8    "attention_types": [9      [10        [11          "global"12        ],13        2814      ]15    ],16    "bos_token_id": 50256,17    "embed_dropout": 0,18    "eos_token_id": 50256,19    "gradient_checkpointing": false,20    "hidden_size": 4096,21    "initializer_range": 0.02,22    "intermediate_size": null,23    "jax": true,24    "layer_norm_epsilon": 1e-05,25    "max_position_embeddings": 2048,26    "model_type": "gpt_neo",27    "num_heads": 16,28    "num_layers": 28,29    "resid_dropout": 0,30    "rotary": true,31    "rotary_dim": 64,32    "summary_activation": null,33    "summary_first_dropout": 0.1,34    "summary_proj_to_labels": true,35    "summary_type": "cls_index",36    "summary_use_proj": true,37    "transformers_version": "4.10.0.dev0",38    "use_cache": true,39    "vocab_size": 50400,40    "window_size": 256,41    "tokenizer_class": "GPT2Tokenizer",42    "task_specific_params": {43      "text-generation": {44        "do_sample": true,45        "temperature": 1.0,46        "max_length": 5047      }48    }49  }