config.json

{
  "att_groups": 8,
  "att_heads": 16,
  "att_query_groups": 8,
  "cross_att_type": "sqa",
  "embed_dim": 512,
  "ff_activation": "silu",
  "ff_dim": 1536,
  "ff_dropout": 0.0,
  "num_layers": 21,
  "rope_base": 100000,
  "self_att_type": "sqa",
  "seq_len": 8192,
  "skip_memory_cross_attention": true,
  "stm_size": 4096,
  "use_attention_output_bias": false,
  "use_flash_attention": true,
  "use_gated": true,
  "use_gated_attention": true,
  "vocab_size": 65536
}