# GPT-2 Small (~124M params)

attention gpt2_small_attn {
    num_heads: 12
    bias: true
    dropout: 0.1
}

ffn gpt2_small_ffn {
    hidden_dim: 3072
    activation: gelu
    bias: true
    dropout: 0.1
}

block gpt2_small_block {
    attention: gpt2_small_attn
    ffn: gpt2_small_ffn
    norm: layernorm { eps: 1e-5 }
    norm_position: pre
    residual: true
}

model gpt2_small {
    description: "GPT-2 Small architecture"
    vocab_size: 32000
    max_seq_len: 1024
    hidden_size: 768
    num_layers: 12
    block: gpt2_small_block
}
