# GPT-2 Large (~774M params)
attention gpt2_large_attn {
    num_heads: 20
    bias: true
    dropout: 0.1
}

ffn gpt2_large_ffn {
    hidden_dim: 5120
    activation: gelu
    bias: true
    dropout: 0.1
}

block gpt2_large_block {
    attention: gpt2_large_attn
    ffn: gpt2_large_ffn
    norm: layernorm { eps: 1e-5 }
    norm_position: pre
    residual: true
}

model gpt2_large {
    description: "GPT-2 Large architecture"
    vocab_size: 32000
    max_seq_len: 1024
    hidden_size: 1280
    num_layers: 36
    block: gpt2_large_block
}
