{
  "models": {
    "gpt-oss-120b": {
      "model_path": "openai/gpt-oss-120b",
      "port": 8001,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--tensor-parallel-size", "4",
        "--max-model-len", "4096",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    },
    "deepseek-r1-70b": {
      "model_path": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
      "port": 8002,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--tensor-parallel-size", "4",
        "--max-model-len", "4096",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    },
    "llama-3.3-70b": {
      "model_path": "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic",
      "port": 8003,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--tensor-parallel-size", "4",
        "--max-model-len", "4096",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    },
    "qwen2.5-72b": {
      "model_path": "RedHatAI/Qwen2.5-72B-Instruct-FP8-dynamic",
      "port": 8004,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--tensor-parallel-size", "4",
        "--max-model-len", "4096",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    },
    "qwen3-32b": {
      "model_path": "Qwen/Qwen3-32B-FP8",
      "port": 8005,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--tensor-parallel-size", "4",
        "--max-model-len", "8192",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    },
    "qwq-32b": {
      "model_path": "RedHatAI/QwQ-32B-FP8-dynamic",
      "port": 8006,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--tensor-parallel-size", "4",
        "--max-model-len", "8192",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    }
  },
  "policy": {
    "policy_type": "time_slice",
    "request_timeout_secs": 600,
    "min_active_secs": 10,
    "max_wait_secs": 60,
    "tick_interval_ms": 500,
    "min_quantum_ms": 5000
  },
  "startup_timeout_secs": 600,
  "max_sleeping": 2,
  "port": 3000,
  "metrics_port": 9090
}
