# LLaMA-style small model (~268M params)
# Uses SwiGLU activation and RMSNorm

attention llama_small_attn {
    num_heads: 16
    bias: false
}

ffn llama_small_ffn {
    hidden_dim: 4096
    activation: swiglu
    bias: false
}

block llama_small_block {
    attention: llama_small_attn
    ffn: llama_small_ffn
    norm: rmsnorm { eps: 1e-5 }
    norm_position: pre
    residual: true
}

model llama_small {
    description: "LLaMA-style architecture with SwiGLU and RMSNorm"
    vocab_size: 32000
    max_seq_len: 2048
    hidden_size: 1024
    num_layers: 16
    block: llama_small_block
}
