{
  "models": {
    "llama-8b": {
      "model_path": "NousResearch/Meta-Llama-3.1-8B-Instruct",
      "port": 8005,
      "eviction": { "weights": "offload", "process": "keep_running" },
      "extra_args": [
        "--max-model-len", "4096",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    }
  },
  "vllm_command": "/home/fergus/.local/bin/vllm",
  "port": 3000,
  "metrics_port": 0
}
