Models and local inference

global:
  llm:
    defaults:
      provider: amazon_bedrock
      model: global.anthropic.claude-sonnet-5
    gateway:
      provider: amazon_bedrock
      model: global.anthropic.claude-sonnet-4-6
    bedrock:
      region: <your Bedrock region>

apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: gpu
spec:
  template:
    metadata:
      labels:
        workload: gpu
    spec:
      nodeClassRef:
        group: eks.amazonaws.com
        kind: NodeClass
        name: default
      requirements:
        - key: eks.amazonaws.com/instance-family
          operator: In
          values: ["g4dn"]
        - key: eks.amazonaws.com/instance-size
          operator: In
          values: ["xlarge", "2xlarge"]
        - key: karpenter.sh/capacity-type
          operator: In
          values: ["on-demand"]
        - key: kubernetes.io/arch
          operator: In
          values: ["amd64"]
      taints:
        - key: nvidia.com/gpu
          value: "present"
          effect: NoSchedule
      terminationGracePeriod: 10m
  limits:
    cpu: "16"
    nvidia.com/gpu: "2"
  disruption:
    consolidationPolicy: WhenEmpty
    consolidateAfter: 10m
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: vllm-models
  namespace: <namespace>
spec:
  accessModes: ["ReadWriteOnce"]
  storageClassName: <storage class>
  resources:
    requests:
      storage: 40Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm
  namespace: <namespace>
  labels: { app: vllm }
spec:
  replicas: 1
  strategy: { type: Recreate }
  selector: { matchLabels: { app: vllm } }
  template:
    metadata:
      labels: { app: vllm }
    spec:
      enableServiceLinks: false   # a Service named vllm would inject VLLM_PORT=tcp://... and stop vLLM
      nodeSelector:
        workload: gpu
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      containers:
        - name: vllm
          image: docker.io/vllm/vllm-openai:v0.9.2   # from your mirror; 0.11 does not run on a T4
          args:
            - --model=Qwen/Qwen2.5-1.5B-Instruct
            - --served-model-name=qwen2.5-1.5b-instruct
            - --dtype=half
            - --max-model-len=32768          # 8192 fails the first turn: the composed prompt is larger
            - --gpu-memory-utilization=0.90
            - --host=0.0.0.0
            - --port=8000
            - --enable-auto-tool-choice     # xpander's Add provider dialog probes for this
            - --tool-call-parser=hermes
          env:
            - name: HF_HOME
              value: /models
            - name: VLLM_API_KEY             # the variable, not --api-key: the flag logs the key
              valueFrom:
                secretKeyRef: { name: vllm-api-key, key: key }
          ports:
            - containerPort: 8000
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: 12Gi
            requests:
              cpu: "2"
              memory: 8Gi
          volumeMounts:
            - { name: models, mountPath: /models }
            - { name: shm, mountPath: /dev/shm }
          readinessProbe:
            httpGet: { path: /health, port: 8000 }
            initialDelaySeconds: 60
            periodSeconds: 15
            failureThreshold: 60
      volumes:
        - name: models
          persistentVolumeClaim: { claimName: vllm-models }
        - name: shm
          emptyDir: { medium: Memory, sizeLimit: 2Gi }
---
apiVersion: v1
kind: Service
metadata:
  name: vllm
  namespace: <namespace>
spec:
  selector: { app: vllm }
  ports:
    - port: 8000
      targetPort: 8000