{
  "models": {
    "llama-8b": {
      "model_path": "NousResearch/Meta-Llama-3.1-8B-Instruct",
      "port": 8005,
      "eviction": { "weights": "retain", "process": "cuda_suspend" },
      "extra_args": [
        "--max-model-len", "4096",
        "--gpu-memory-utilization", "0.85",
        "--enforce-eager"
      ]
    }
  },
  "checkpoint": {
    "criu_path": "/tmp/criu/criu/criu",
    "cuda_plugin_dir": "/tmp/criu/plugins/cuda/",
    "images_dir": "/tmp/llmux-checkpoints",
    "cuda_checkpoint_path": "cuda-checkpoint"
  },
  "vllm_command": "/home/fergus/.local/bin/vllm",
  "port": 3000,
  "metrics_port": 0
}
