An inference request and each attempt at an engine carry gen_ai.*: the operation, the requested and answering model, max_tokens, temperature and top_p when the client set them, the session as the conversation id, the input, output and cached tokens, and on an attempt the engine that answered. A customer's Grafana, Jaeger or Langfuse then shows llmgw's calls as model calls. No prompt or answer text is ever put in a span, and a test says so. BACKLOG gains section L: what is worth taking from Kong's AI Gateway, of which this is the first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
38dfe268c9
commit
06baa38858
7 files changed
+263
-1
No files matched your search
@@ -450,6 +450,8 @@ type parsedRequest struct {
|
||||
// longest one is what a context must hold.
|
||||
ContextTokens int
|
||||
MaxTokens int
|
||||
// Temperature and TopP are the client's, when it set them: for the trace.
|
||||
Temperature, TopP *float64
|
||||
|
||||
// ThreadID is the conversation this request continues, empty for the
|
||||
// stateless path every request took before threads existed.
|
||||
@@ -505,7 +507,10 @@ type inferenceBody struct {
|
||||
Input json.RawMessage `json:"input"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
MaxCompletionTokens int `json:"max_completion_tokens"`
|
||||
StreamOptions *struct {
|
||||
// Read only to say them in the request's trace; sent on untouched.
|
||||
Temperature *float64 `json:"temperature"`
|
||||
TopP *float64 `json:"top_p"`
|
||||
StreamOptions *struct {
|
||||
IncludeUsage bool `json:"include_usage"`
|
||||
} `json:"stream_options"`
|
||||
}
|
||||
@@ -524,6 +529,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
parsed, err := r.parse(req, path)
|
||||
if err == nil {
|
||||
span.SetAttributes(attribute.String("llmgw.model", parsed.Model))
|
||||
span.SetAttributes(genAIRequest(path, parsed)...)
|
||||
parsed.RequestID = rid
|
||||
if parsed.Header == nil {
|
||||
parsed.Header = http.Header{}
|
||||
@@ -688,6 +694,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
span.SetAttributes(attribute.Int("http.response.status_code", status),
|
||||
attribute.String("llmgw.served_model", res.model),
|
||||
attribute.String("llmgw.endpoint", res.endpointID))
|
||||
span.SetAttributes(genAIUsage(path, res.model, res.outcome.Usage)...)
|
||||
if status >= 500 {
|
||||
span.SetStatus(codes.Error, http.StatusText(status))
|
||||
}
|
||||
@@ -891,6 +898,8 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
Band: req.Header.Get(r.cfg.Admission.BandHeader),
|
||||
ProfileOverride: req.Header.Get("x-llmgw-profile"),
|
||||
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
|
||||
Temperature: b.Temperature,
|
||||
TopP: b.TopP,
|
||||
PromptPrefix: promptPrefix(&b),
|
||||
ThreadID: req.Header.Get(threadHeader),
|
||||
}
|
||||
|
||||
@@ -1183,6 +1183,11 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
uctx, uspan := tracer().Start(ctx, "upstream", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID),
|
||||
attribute.String("url.full", dec.Endpoint.URL+req.Path)))
|
||||
if op := genAIOperation(req.Path); op != "" {
|
||||
uspan.SetAttributes(attribute.String("gen_ai.operation.name", op),
|
||||
attribute.String("gen_ai.provider.name", genAIProvider(dec.Endpoint.Engine)),
|
||||
attribute.String("gen_ai.request.model", model))
|
||||
}
|
||||
body := req.UpstreamBody
|
||||
// A shared ComfyUI replica is told which of its models' checkpoint;
|
||||
// any other endpoint its own upstream name.
|
||||
@@ -1256,6 +1261,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
}
|
||||
releaseAdapter()
|
||||
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
|
||||
uspan.SetAttributes(genAIUsage(req.Path, model, out.Usage)...)
|
||||
endSpan(uspan, perr)
|
||||
adapterGone := onDemand && !out.Wrote && out.Status == http.StatusNotFound
|
||||
if adapterGone {
|
||||
|
||||
@@ -3,12 +3,15 @@ package router
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
)
|
||||
|
||||
// tracer is looked up per use rather than kept, so a provider installed after
|
||||
@@ -49,3 +52,82 @@ func withTraceparent(ctx context.Context, h http.Header) http.Header {
|
||||
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(out))
|
||||
return out
|
||||
}
|
||||
|
||||
// The OpenTelemetry GenAI semantic conventions: the names a trace backend
|
||||
// reads an LLM call by, so llmgw's spans show in a customer's own tools —
|
||||
// Grafana, Jaeger, Langfuse, anything that speaks OpenTelemetry — as model
|
||||
// calls with their model and tokens, without a dashboard made for llmgw.
|
||||
//
|
||||
// What a request said and what the model answered are not among them, and
|
||||
// are never added: llmgw keeps no prompt text anywhere, and a trace is the
|
||||
// last place one should leak into.
|
||||
|
||||
// genAIOperation is the conventions' operation for an inference path, or ""
|
||||
// for one they have no name for. The Anthropic Messages and Responses APIs
|
||||
// are answered as chat completions, and so are chat.
|
||||
func genAIOperation(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, "/chat/completions"):
|
||||
return "chat"
|
||||
case strings.HasSuffix(path, "/completions"):
|
||||
return "text_completion"
|
||||
case strings.HasSuffix(path, "/embeddings"):
|
||||
return "embeddings"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// genAIRequest is what a request asked for, as the conventions name it.
|
||||
func genAIRequest(path string, p *parsedRequest) []attribute.KeyValue {
|
||||
op := genAIOperation(path)
|
||||
if op == "" {
|
||||
return nil
|
||||
}
|
||||
kv := []attribute.KeyValue{
|
||||
attribute.String("gen_ai.operation.name", op),
|
||||
attribute.String("gen_ai.request.model", p.Model),
|
||||
}
|
||||
if p.MaxTokens > 0 {
|
||||
kv = append(kv, attribute.Int("gen_ai.request.max_tokens", p.MaxTokens))
|
||||
}
|
||||
if p.Temperature != nil {
|
||||
kv = append(kv, attribute.Float64("gen_ai.request.temperature", *p.Temperature))
|
||||
}
|
||||
if p.TopP != nil {
|
||||
kv = append(kv, attribute.Float64("gen_ai.request.top_p", *p.TopP))
|
||||
}
|
||||
if p.SessionID != "" {
|
||||
kv = append(kv, attribute.String("gen_ai.conversation.id", p.SessionID))
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// genAIUsage is what an answer cost, in the conventions' names: the model
|
||||
// that answered, and its tokens, cached ones apart.
|
||||
func genAIUsage(path, served string, u *proxy.Usage) []attribute.KeyValue {
|
||||
if genAIOperation(path) == "" {
|
||||
return nil
|
||||
}
|
||||
var kv []attribute.KeyValue
|
||||
if served != "" {
|
||||
kv = append(kv, attribute.String("gen_ai.response.model", served))
|
||||
}
|
||||
if u != nil {
|
||||
kv = append(kv,
|
||||
attribute.Int("gen_ai.usage.input_tokens", u.PromptTokens),
|
||||
attribute.Int("gen_ai.usage.output_tokens", u.CompletionTokens))
|
||||
if u.CachedTokens > 0 {
|
||||
kv = append(kv, attribute.Int("gen_ai.usage.cache_read.input_tokens", u.CachedTokens))
|
||||
}
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// genAIProvider names who answered an attempt: the engine serving a replica
|
||||
// (vllm, llamacpp, …), or the outside provider's, for an external model.
|
||||
func genAIProvider(engine string) string {
|
||||
if engine == "" {
|
||||
return "llmgw"
|
||||
}
|
||||
return engine
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
@@ -81,3 +82,62 @@ func TestWithoutTracingTheEngineGetsNoTraceparent(t *testing.T) {
|
||||
t.Errorf("traceparent %q sent with tracing off", tp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracesSpeakTheGenAIConventionsAndHoldNoText(t *testing.T) {
|
||||
rec := tracetest.NewSpanRecorder()
|
||||
prevTP := otel.GetTracerProvider()
|
||||
otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)))
|
||||
t.Cleanup(func() { otel.SetTracerProvider(prevTP) })
|
||||
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
|
||||
res := post(t, rt.Handler(), map[string]any{"model": "m", "max_tokens": 64, "temperature": 0.2, "top_p": 0.9,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "the secret word is aubergine"}}}, nil)
|
||||
if res.Code != 200 {
|
||||
t.Fatalf("status %d: %s", res.Code, res.Body.String())
|
||||
}
|
||||
got := map[string]map[string]string{}
|
||||
for _, s := range rec.Ended() {
|
||||
attrs := map[string]string{}
|
||||
for _, a := range s.Attributes() {
|
||||
attrs[string(a.Key)] = a.Value.Emit()
|
||||
// Neither the prompt nor the reply ("hi"), in any attribute.
|
||||
if v := a.Value.Emit(); strings.Contains(v, "aubergine") || v == "hi" {
|
||||
t.Errorf("span %s carries text in %s", s.Name(), a.Key)
|
||||
}
|
||||
}
|
||||
got[s.Name()] = attrs
|
||||
}
|
||||
root := got["inference /v1/chat/completions"]
|
||||
for k, want := range map[string]string{
|
||||
"gen_ai.operation.name": "chat",
|
||||
"gen_ai.request.model": "m",
|
||||
"gen_ai.request.max_tokens": "64",
|
||||
"gen_ai.request.temperature": "0.2",
|
||||
"gen_ai.request.top_p": "0.9",
|
||||
"gen_ai.response.model": "m",
|
||||
"gen_ai.usage.input_tokens": "4",
|
||||
"gen_ai.usage.output_tokens": "2",
|
||||
} {
|
||||
if root[k] != want {
|
||||
t.Errorf("request span %s = %q, want %q", k, root[k], want)
|
||||
}
|
||||
}
|
||||
up := got["upstream"]
|
||||
if up["gen_ai.operation.name"] != "chat" || up["gen_ai.provider.name"] == "" || up["gen_ai.usage.output_tokens"] != "2" {
|
||||
t.Errorf("upstream span %v", up)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryInferencePathHasItsGenAIOperation(t *testing.T) {
|
||||
for path, want := range map[string]string{
|
||||
"/v1/chat/completions": "chat",
|
||||
"/v1/completions": "text_completion",
|
||||
"/v1/embeddings": "embeddings",
|
||||
"/v1/rerank": "",
|
||||
} {
|
||||
if got := genAIOperation(path); got != want {
|
||||
t.Errorf("%s: %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user