kind: Container
metadata:
  name: vllm-qwen
  labels:
    type: inference
platform: runpod
accelerators:
  - "1:L4"
image: "vllm/vllm-openai:latest"
command: |
  env && python3 -m vllm.entrypoints.openai.api_server \
    --model $MODEL \
    --port 8000 \
    --host 0.0.0.0 \
    --gpu-memory-utilization 0.8 \
    --max-model-len 2048 \
    --enable-lora \
    --max-num-seqs 1
env:
  - key: MODEL
    value: Qwen/Qwen2.5-0.5B-Instruct
  - key: HF_TOKEN
    secret_name: HF_TOKEN
  - key: VLLM_ALLOW_RUNTIME_LORA_UPDATING
    value: True
restart: Always
proxy_port: 8000
authz:
  rules:
    # Only allow adapters to be loaded based on the user's organization name or handle
    - name: model-access
      field_match:
        - json_path: "$.model"
          pattern: "${org_name}/**"
        - json_path: "$.model"
          pattern: "${handle}/**"
      allow: true
meters:
  - metric: response_value
    json_path: "$.usage.completion_tokens"
    unit: token
    cost: 0.000001
    currency: USD
