family: qwen3
display_name: "Qwen3"
vendor: Alibaba
architectures:
  - Qwen3ForCausalLM
hf_pattern: "Qwen/Qwen3*"

size_variants:
  0.6b:
    parameters: "0.6B"
    hidden_dim: 1024
    num_layers: 28
    num_heads: 16
    num_kv_heads: 8
    intermediate_dim: 3072
    vocab_size: 151936
    max_position_embeddings: 40960
    head_dim: 128
    rope_theta: 1000000.0
    rms_norm_eps: 0.000001
    tied_embeddings: true
  1.7b:
    parameters: "1.7B"
    hidden_dim: 2048
    num_layers: 28
    num_heads: 16
    num_kv_heads: 8
    intermediate_dim: 6144
    vocab_size: 151936
    max_position_embeddings: 40960
    head_dim: 128
    rope_theta: 1000000.0
    rms_norm_eps: 0.000001
    tied_embeddings: true
  4b:
    parameters: "4B"
    hidden_dim: 2560
    num_layers: 36
    num_heads: 32
    num_kv_heads: 8
    intermediate_dim: 9728
    vocab_size: 151936
    max_position_embeddings: 40960
    head_dim: 128
    rope_theta: 1000000.0
    rms_norm_eps: 0.000001
    tied_embeddings: true
  8b:
    parameters: "8B"
    hidden_dim: 4096
    num_layers: 36
    num_heads: 32
    num_kv_heads: 8
    intermediate_dim: 12288
    vocab_size: 151936
    max_position_embeddings: 40960
    head_dim: 128
    rope_theta: 1000000.0
    rms_norm_eps: 0.000001
  14b:
    parameters: "14B"
    hidden_dim: 5120
    num_layers: 40
    num_heads: 40
    num_kv_heads: 8
    intermediate_dim: 17408
    vocab_size: 151936
    max_position_embeddings: 40960
    head_dim: 128
    rope_theta: 1000000.0
    rms_norm_eps: 0.000001
  # NOTE: Qwen3-30B-A3B is MoE (Qwen3MoeForCausalLM), not dense — excluded from this contract
  32b:
    parameters: "32B"
    hidden_dim: 5120
    num_layers: 64
    num_heads: 64
    num_kv_heads: 8
    intermediate_dim: 25600
    vocab_size: 151936
    max_position_embeddings: 40960
    head_dim: 128
    rope_theta: 1000000.0
    rms_norm_eps: 0.000001

# =============================================================================
# CUDA TRAINING RESOURCE BUDGET (Provable Design-by-Contract)
#
# Meyer (1992) "No Hidden Clauses": max_seq_len MUST be derived from model dims
# and available VRAM — never hardcoded. Every buffer allocation in
# entrenar/src/transformer/cuda_block.rs is a known function of these dims.
#
# FORMULA (f32, 4 bytes per element):
#
#   Let H = hidden_dim, Q = num_heads * head_dim, KV = num_kv_heads * head_dim,
#       I = intermediate_dim, N = num_heads, hd = head_dim, L = num_layers,
#       S = max_seq_len
#
#   per_layer_weights = Q*H + KV*H*2 + H*Q + I*H*3 + H*2          (constant)
#   per_layer_grad_weights = H*H*2 + H*KV*2 + H*I*3 + H*2         (constant)
#   per_layer_scratch_linear(S) = S*(8H + 2KV + 4I + 3N*hd)        (linear in S)
#   per_layer_scratch_quadratic(S) = N*S*max(S,hd) + N*S*S          (quadratic in S)
#
#   total_vram_bytes(S) = 4 * L * (per_layer_weights + per_layer_grad_weights
#                                   + per_layer_scratch_linear(S)
#                                   + per_layer_scratch_quadratic(S))
#
# RESOLUTION ORDER (highest priority first):
#   1. User CLI override: --max-seq-len <N>
#   2. Data-derived: max token count in training corpus
#   3. VRAM budget solver: largest S such that total_vram_bytes(S) <= available_vram
#
# INVARIANT: max_seq_len <= max_position_embeddings (always)
# INVARIANT: max_seq_len >= max_token_count_in_data (correctness)
# =============================================================================

constraints:
  attention_type: gqa
  activation: silu
  norm_type: rmsnorm
  has_bias: false
  # NOTE: tied_embeddings varies by size — 0.6B/1.7B/4B=true, 8B/14B/32B=false
  # Per-size override in size_variants where tied_embeddings=true
  tied_embeddings: false
  positional_encoding: rope
  mlp_type: swiglu
  qk_norm: "true"

tensor_template:
  embedding: "model.embed_tokens.weight"
  lm_head: "lm_head.weight"
  final_norm: "model.norm.weight"
  per_layer:
    q_proj: "model.layers.{n}.self_attn.q_proj.weight"
    k_proj: "model.layers.{n}.self_attn.k_proj.weight"
    v_proj: "model.layers.{n}.self_attn.v_proj.weight"
    o_proj: "model.layers.{n}.self_attn.o_proj.weight"
    gate_proj: "model.layers.{n}.mlp.gate_proj.weight"
    up_proj: "model.layers.{n}.mlp.up_proj.weight"
    down_proj: "model.layers.{n}.mlp.down_proj.weight"
    input_layernorm: "model.layers.{n}.input_layernorm.weight"
    post_attention_layernorm: "model.layers.{n}.post_attention_layernorm.weight"
    q_norm: "model.layers.{n}.self_attn.q_norm.weight"
    k_norm: "model.layers.{n}.self_attn.k_norm.weight"

shape_template:
  embedding: "[vocab_size, hidden_dim]"
  lm_head: "[vocab_size, hidden_dim]"
  q_proj: "[num_heads * head_dim, hidden_dim]"
  k_proj: "[num_kv_heads * head_dim, hidden_dim]"
  v_proj: "[num_kv_heads * head_dim, hidden_dim]"
  o_proj: "[hidden_dim, num_heads * head_dim]"
  gate_proj: "[intermediate_dim, hidden_dim]"
  up_proj: "[intermediate_dim, hidden_dim]"
  down_proj: "[hidden_dim, intermediate_dim]"
  input_layernorm: "[hidden_dim]"
  post_attention_layernorm: "[hidden_dim]"
  q_norm: "[head_dim]"
  k_norm: "[head_dim]"

quantizations:
  - q4_k_m
  - q5_k_m
  - q6_k
  - q8_0
  - f16
  - f32

chat_template:
  format: chatml
  template: "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
  bos_token: ""
  eos_token: "<|im_end|>"
  special_tokens:
    im_start: "<|im_start|>"
    im_end: "<|im_end|>"
    endoftext: "<|endoftext|>"
    think_start: "<think>"
    think_end: "</think>"

certification:
  playbook_path: "../apr-model-qa-playbook/playbooks/models/qwen3-{size}.playbook.yaml"
  csv_family_key: "qwen3"
  size_categories:
    0.6b: tiny
    1.7b: small
    4b: small
    8b: medium
    14b: large
    32b: xlarge
