Files
llmgw/internal/sim/metrics.go
T
ilkerandClaude Opus 5 e07bc46c57
CI / test (push) Has been cancelled
Faz 0: iskelet ve sim-engine
Plan v0.2'nin Faz 0 çıktısı. Yazılan sıraya göre:

- cobra tabanlı CLI iskeleti; server/agent/ctl komutları henüz stub.
- internal/sim: OpenAI uyumlu sahte inference engine. Router'ı GPU olmadan
  gerçekçi koşullarda test edebilmek için gerçek bir engine'in router
  açısından önemli davranışlarını modeller:
    * continuous batching ve KV blok muhasebesi (head-of-line blocking dahil)
    * zincirlenmiş blok hash'leri + LRU ile otomatik prefix cache
    * batch büyüdükçe artan token gecikmesi
    * istemci koptuğunda isteğin düşürülüp blokların bırakılması
    * vLLM metrik adlarıyla /metrics (plan §7.6 engine kontratı)
- Testler: prefix hash zincirlemesi, LRU tahliyesi, iptal propagasyonu,
  kuyruklama, SSE formatı, usage/cached_tokens, metriklerin gerçek
  Prometheus parser'ıyla okunabilirliği.
- Gitea Actions CI: gofmt, vet, race detector.

Modül yolu gitea.mynodes.xyz/ilker/llmgw; `make rename` ile değiştirilebilir.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:00:03 +02:00

143 lines
5.1 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package sim
import (
"fmt"
"io"
"sync"
"sync/atomic"
"time"
)
// ttftBuckets ve tpotBuckets, vLLM'in kullandıklarına yakın saniye sınırları.
var (
ttftBuckets = []float64{0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 20, 40, 80}
tpotBuckets = []float64{0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1, 2.5}
)
// histogram, Prometheus uyumlu basit bir kümülatif histogram.
type histogram struct {
bounds []float64
counts []uint64
sum float64
count uint64
}
func newHistogram(bounds []float64) *histogram {
return &histogram{bounds: bounds, counts: make([]uint64, len(bounds))}
}
func (h *histogram) observe(v float64) {
h.sum += v
h.count++
for i, b := range h.bounds {
if v <= b {
h.counts[i]++
}
}
}
// write, histogramı Prometheus text formatında yazar (kümülatif le bucket'ları).
func (h *histogram) write(w io.Writer, name, labels string) {
var cum uint64
for i, b := range h.bounds {
cum = h.counts[i]
fmt.Fprintf(w, "%s_bucket{%s,le=\"%g\"} %d\n", name, labels, b, cum)
}
fmt.Fprintf(w, "%s_bucket{%s,le=\"+Inf\"} %d\n", name, labels, h.count)
fmt.Fprintf(w, "%s_sum{%s} %g\n", name, labels, h.sum)
fmt.Fprintf(w, "%s_count{%s} %d\n", name, labels, h.count)
}
type metrics struct {
received atomic.Int64
finished atomic.Int64
failed atomic.Int64
cancelled atomic.Int64
promptTokens atomic.Int64
genTokens atomic.Int64
mu sync.Mutex
ttft *histogram
tpot *histogram
}
func newMetrics() *metrics {
return &metrics{
ttft: newHistogram(ttftBuckets),
tpot: newHistogram(tpotBuckets),
}
}
func (m *metrics) observeTTFT(d time.Duration) {
m.mu.Lock()
m.ttft.observe(d.Seconds())
m.mu.Unlock()
}
func (m *metrics) observeTPOT(d time.Duration) {
m.mu.Lock()
m.tpot.observe(d.Seconds())
m.mu.Unlock()
}
// WriteMetrics, engine durumunu vLLM'in metrik adlarıyla Prometheus text
// formatında yazar.
//
// Metrik adları §7.6'daki engine kontratıdır; router bunlara göre parse eder.
// Gerçek vLLM sürümleriyle aynı adları kullanmak şart — sim-engine'in varlık
// sebebi router'ın gerçek engine'e karşı davranışını taklit etmek.
func (e *Engine) WriteMetrics(w io.Writer) {
s := e.Stats()
lbl := fmt.Sprintf("model_name=%q", e.cfg.Model)
fmt.Fprintf(w, "# HELP vllm:num_requests_running Number of requests currently running on GPU.\n")
fmt.Fprintf(w, "# TYPE vllm:num_requests_running gauge\n")
fmt.Fprintf(w, "vllm:num_requests_running{%s} %d\n", lbl, s.Running)
fmt.Fprintf(w, "# HELP vllm:num_requests_waiting Number of requests waiting to be processed.\n")
fmt.Fprintf(w, "# TYPE vllm:num_requests_waiting gauge\n")
fmt.Fprintf(w, "vllm:num_requests_waiting{%s} %d\n", lbl, s.Waiting)
fmt.Fprintf(w, "# HELP vllm:kv_cache_usage_perc KV-cache usage. 1 means 100 percent usage.\n")
fmt.Fprintf(w, "# TYPE vllm:kv_cache_usage_perc gauge\n")
fmt.Fprintf(w, "vllm:kv_cache_usage_perc{%s} %g\n", lbl, s.KVCacheUsage)
fmt.Fprintf(w, "# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n")
fmt.Fprintf(w, "# TYPE vllm:cache_config_info gauge\n")
fmt.Fprintf(w, "vllm:cache_config_info{block_size=\"%d\",num_gpu_blocks=\"%d\"} 1\n", s.BlockSize, s.NumGPUBlocks)
fmt.Fprintf(w, "# HELP vllm:prefix_cache_queries_total Prefix cache queries, in terms of number of queried blocks.\n")
fmt.Fprintf(w, "# TYPE vllm:prefix_cache_queries_total counter\n")
fmt.Fprintf(w, "vllm:prefix_cache_queries_total{%s} %d\n", lbl, s.PrefixQueries)
fmt.Fprintf(w, "# HELP vllm:prefix_cache_hits_total Prefix cache hits, in terms of number of cached blocks.\n")
fmt.Fprintf(w, "# TYPE vllm:prefix_cache_hits_total counter\n")
fmt.Fprintf(w, "vllm:prefix_cache_hits_total{%s} %d\n", lbl, s.PrefixHits)
fmt.Fprintf(w, "# HELP vllm:prompt_tokens_total Number of prefill tokens processed.\n")
fmt.Fprintf(w, "# TYPE vllm:prompt_tokens_total counter\n")
fmt.Fprintf(w, "vllm:prompt_tokens_total{%s} %d\n", lbl, e.m.promptTokens.Load())
fmt.Fprintf(w, "# HELP vllm:generation_tokens_total Number of generation tokens processed.\n")
fmt.Fprintf(w, "# TYPE vllm:generation_tokens_total counter\n")
fmt.Fprintf(w, "vllm:generation_tokens_total{%s} %d\n", lbl, e.m.genTokens.Load())
fmt.Fprintf(w, "# HELP vllm:request_success_total Count of successfully processed requests.\n")
fmt.Fprintf(w, "# TYPE vllm:request_success_total counter\n")
fmt.Fprintf(w, "vllm:request_success_total{%s,finished_reason=\"length\"} %d\n", lbl, e.m.finished.Load())
// Simülasyona özgü (gerçek vLLM'de yok; kaos testlerinde işe yarar).
fmt.Fprintf(w, "# TYPE sim:requests_cancelled_total counter\n")
fmt.Fprintf(w, "sim:requests_cancelled_total{%s} %d\n", lbl, e.m.cancelled.Load())
fmt.Fprintf(w, "# TYPE sim:requests_failed_total counter\n")
fmt.Fprintf(w, "sim:requests_failed_total{%s} %d\n", lbl, e.m.failed.Load())
e.m.mu.Lock()
defer e.m.mu.Unlock()
fmt.Fprintf(w, "# TYPE vllm:time_to_first_token_seconds histogram\n")
e.m.ttft.write(w, "vllm:time_to_first_token_seconds", lbl)
fmt.Fprintf(w, "# TYPE vllm:time_per_output_token_seconds histogram\n")
e.m.tpot.write(w, "vllm:time_per_output_token_seconds", lbl)
}