第 13 章 可观测性与成本优化

系统 throughput = 并发请求数 × 单请求 TPS

# 从 vLLM 的 metrics 端点获取
curl http://vllm-server:8000/metrics | grep vllm_num_requests
# vllm:num_requests_running 8
# vllm:num_requests_waiting 3  <-- 有 3 个在排队

# Prometheus scrape config
scrape_configs:
  - job_name: 'vllm'
    metrics_path: '/metrics'
    scrape_interval: 15s
    static_configs:
      - targets: ['vllm-server:8000']

[API Gateway]
  └── [Auth + Rate Limit Check]       ~2ms
  └── [Model Router]                   ~1ms
  └── [Backend Request]
       └── [Queue Wait]               可能几秒
       └── [Prefill]                   跟输入长度正相关
       └── [Decode]                    跟输出长度正相关
  └── [Response Streaming]             持续时间 = output_tokens / TPS

from opentelemetry import trace
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 
# 初始化
provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(endpoint="http://otel-collector:4317"))
)
trace.set_tracer_provider(provider)
tracer = trace.get_tracer(__name__)
 
# 自动注入 FastAPI
FastAPIInstrumentor.instrument_app(app)
 
# 手动添加 LLM 特有的 span 属性
@app.post("/v1/chat/completions")
async def chat_completions(request: ChatRequest):
    with tracer.start_as_current_span("llm_request") as span:
        span.set_attribute("llm.model", request.model)
        span.set_attribute("llm.input_tokens", count_tokens(request.messages))
 
        with tracer.start_as_current_span("model_routing"):
            backend = router.select(request)
 
        with tracer.start_as_current_span("backend_call") as backend_span:
            backend_span.set_attribute("backend.url", backend)
            response = await call_backend(backend, request)
 
        span.set_attribute("llm.output_tokens", response.usage.completion_tokens)
        span.set_attribute("llm.total_tokens", response.usage.total_tokens)
        return response

# docker-compose 快速部署
services:
  tempo:
    image: grafana/tempo:2.6.1
    command: ["-config.file=/etc/tempo.yaml"]
    volumes:
      - ./tempo.yaml:/etc/tempo.yaml
      - tempo-data:/var/tempo
    ports:
      - "4317:4317"   # OTLP gRPC
      - "3200:3200"   # Tempo API
 
  grafana:
    image: grafana/grafana:11.4.0
    ports:
      - "3000:3000"
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true

1500 × 0.7 × 3600 × 24 × 30 ≈ 27 亿 tokens/月

自建成本：¥2.5/百万 tokens（固定成本，不管用不用都要付）
云 API：  ¥0.35/百万 tokens（SiliconFlow 7B，按量付费）

def break_even_analysis(
    gpu_monthly_cost: float,      # GPU 月租金
    max_throughput: float,         # 最大吞吐（tokens/s）
    utilization: float,            # 平均利用率
    cloud_price_per_mtok: float,   # 云 API 每百万 token 价格
) -> dict:
    monthly_tokens = max_throughput * utilization * 3600 * 24 * 30
    self_hosted_per_mtok = gpu_monthly_cost / (monthly_tokens / 1_000_000)
 
    break_even_util = gpu_monthly_cost / (
        cloud_price_per_mtok * max_throughput * 3600 * 24 * 30 / 1_000_000
    )
 
    return {
        "self_hosted_per_mtok": round(self_hosted_per_mtok, 2),
        "cloud_per_mtok": cloud_price_per_mtok,
        "break_even_utilization": f"{break_even_util:.1%}",
        "recommendation": "self-hosted" if self_hosted_per_mtok < cloud_price_per_mtok else "cloud"
    }

                    ┌─────────────┐
                    │  API Gateway │
                    └──────┬──────┘
                           │
              ┌────────────┴────────────┐
              │                         │
    ┌─────────▼─────────┐   ┌──────────▼──────────┐
    │  自建 GPU 集群      │   │  云 API (溢出)       │
    │  处理基线流量        │   │  处理峰值流量         │
    │  2 × A10 实例      │   │  SiliconFlow/阿里通义 │
    └───────────────────┘   └─────────────────────┘

# Prometheus Adapter 配置
rules:
  - seriesQuery: 'vllm:num_requests_waiting{namespace!="",pod!=""}'
    resources:
      overrides:
        namespace: {resource: "namespace"}
        pod: {resource: "pod"}
    name:
      matches: "vllm:num_requests_waiting"
      as: "vllm_pending_requests"
    metricsQuery: 'avg(vllm:num_requests_waiting{<<.LabelMatchers>>})'

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: vllm-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: vllm-qwen-7b
  minReplicas: 2
  maxReplicas: 8
  metrics:
  - type: Pods
    pods:
      metric:
        name: vllm_pending_requests
      target:
        type: AverageValue
        averageValue: "5"  # 每个 Pod 平均排队 5 个请求时扩容
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60   # 1 分钟窗口，避免抖动
      policies:
      - type: Pods
        value: 2
        periodSeconds: 60              # 每次最多加 2 个 Pod
    scaleDown:
      stabilizationWindowSeconds: 300  # 5 分钟窗口，缩容要谨慎
      policies:
      - type: Pods
        value: 1
        periodSeconds: 120             # 每 2 分钟最多缩 1 个

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: vllm-scaledobject
spec:
  scaleTargetRef:
    name: vllm-qwen-7b
  minReplicaCount: 2
  maxReplicaCount: 8
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus:9090
      metricName: vllm_pending_requests
      query: |
        avg(vllm:num_requests_waiting{deployment="vllm-qwen-7b"})
      threshold: "5"
  advanced:
    horizontalPodAutoscalerConfig:
      behavior:
        scaleDown:
          stabilizationWindowSeconds: 300

# 启动时开启
python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen2.5-7B-Instruct \
  --enable-prefix-caching