The Complete Production SOP: How to Deploy and Run Aphrodite Engine Distributed LLM Serving in Autonomous AI Agents

---
name: "aphrodite-engine-inference-service"
description: "Serve large language models with diverse quantization formats using Aphrodite Engine."
metadata:
  author: "PygmalionAI"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "deploy aphrodite engine"
  - "serve gptq model aphrodite"
  - "run marlin kernel inference"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

#!/usr/bin/env bash
python3 -m aphrodite.endpoints.openai.api_server \
    --model "TheBloke/Llama-2-13B-chat-GPTQ" \
    --quantization marlin \
    --port 2242 \
    --gpu-memory-utilization 0.90 \
    --max-num-batched-tokens 4096

import openai

client = openai.OpenAI(
    base_url="http://localhost:2242/v1",
    api_key="aphrodite-internal",
)

response = client.chat.completions.create(
    model="TheBloke/Llama-2-13B-chat-GPTQ",
    messages=[{"role": "user", "content": "Verify Aphrodite inference latency."}],
    temperature=0.2,
    max_tokens=64,
)
print(response.choices[0].message.content)

#!/usr/bin/env bash
HEALTH_URL="http://localhost:2242/health"
STATUS=$(curl -o /dev/null -s -w "%{http_code}" "${HEALTH_URL}")
if [ "$STATUS" -eq 200 ]; then
  echo "Aphrodite Engine ready for distributed routing."
  exit 0
else
  echo "Service degraded with status: ${STATUS}"
  exit 1
fi