Traces say model calls in the OpenTelemetry GenAI conventions
CI / test (push) Successful in 8m55s

An inference request and each attempt at an engine carry gen_ai.*: the
operation, the requested and answering model, max_tokens, temperature and
top_p when the client set them, the session as the conversation id, the
input, output and cached tokens, and on an attempt the engine that answered.
A customer's Grafana, Jaeger or Langfuse then shows llmgw's calls as model
calls. No prompt or answer text is ever put in a span, and a test says so.

BACKLOG gains section L: what is worth taking from Kong's AI Gateway, of
which this is the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-25 12:59:20 +01:00
1 parent 38dfe268c9
commit 06baa38858
7 files changed
+263 -1

No files matched your search

+10 -1
View File
@@ -450,6 +450,8 @@ type parsedRequest struct {
// longest one is what a context must hold.
ContextTokens int
MaxTokens int
// Temperature and TopP are the client's, when it set them: for the trace.
Temperature, TopP *float64
// ThreadID is the conversation this request continues, empty for the
// stateless path every request took before threads existed.
@@ -505,7 +507,10 @@ type inferenceBody struct {
Input json.RawMessage `json:"input"`
MaxTokens int `json:"max_tokens"`
MaxCompletionTokens int `json:"max_completion_tokens"`
StreamOptions *struct {
// Read only to say them in the request's trace; sent on untouched.
Temperature *float64 `json:"temperature"`
TopP *float64 `json:"top_p"`
StreamOptions *struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
}
@@ -524,6 +529,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
parsed, err := r.parse(req, path)
if err == nil {
span.SetAttributes(attribute.String("llmgw.model", parsed.Model))
span.SetAttributes(genAIRequest(path, parsed)...)
parsed.RequestID = rid
if parsed.Header == nil {
parsed.Header = http.Header{}
@@ -688,6 +694,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
span.SetAttributes(attribute.Int("http.response.status_code", status),
attribute.String("llmgw.served_model", res.model),
attribute.String("llmgw.endpoint", res.endpointID))
span.SetAttributes(genAIUsage(path, res.model, res.outcome.Usage)...)
if status >= 500 {
span.SetStatus(codes.Error, http.StatusText(status))
}
@@ -891,6 +898,8 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
Band: req.Header.Get(r.cfg.Admission.BandHeader),
ProfileOverride: req.Header.Get("x-llmgw-profile"),
MaxTokens: max(b.MaxTokens, b.MaxCompletionTokens),
Temperature: b.Temperature,
TopP: b.TopP,
PromptPrefix: promptPrefix(&b),
ThreadID: req.Header.Get(threadHeader),
}
+6
View File
@@ -1183,6 +1183,11 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
uctx, uspan := tracer().Start(ctx, "upstream", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID),
attribute.String("url.full", dec.Endpoint.URL+req.Path)))
if op := genAIOperation(req.Path); op != "" {
uspan.SetAttributes(attribute.String("gen_ai.operation.name", op),
attribute.String("gen_ai.provider.name", genAIProvider(dec.Endpoint.Engine)),
attribute.String("gen_ai.request.model", model))
}
body := req.UpstreamBody
// A shared ComfyUI replica is told which of its models' checkpoint;
// any other endpoint its own upstream name.
@@ -1256,6 +1261,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
}
releaseAdapter()
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
uspan.SetAttributes(genAIUsage(req.Path, model, out.Usage)...)
endSpan(uspan, perr)
adapterGone := onDemand && !out.Wrote && out.Status == http.StatusNotFound
if adapterGone {
+82
View File
@@ -3,12 +3,15 @@ package router
import (
"context"
"net/http"
"strings"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// tracer is looked up per use rather than kept, so a provider installed after
@@ -49,3 +52,82 @@ func withTraceparent(ctx context.Context, h http.Header) http.Header {
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(out))
return out
}
// The OpenTelemetry GenAI semantic conventions: the names a trace backend
// reads an LLM call by, so llmgw's spans show in a customer's own tools —
// Grafana, Jaeger, Langfuse, anything that speaks OpenTelemetry — as model
// calls with their model and tokens, without a dashboard made for llmgw.
//
// What a request said and what the model answered are not among them, and
// are never added: llmgw keeps no prompt text anywhere, and a trace is the
// last place one should leak into.
// genAIOperation is the conventions' operation for an inference path, or ""
// for one they have no name for. The Anthropic Messages and Responses APIs
// are answered as chat completions, and so are chat.
func genAIOperation(path string) string {
switch {
case strings.HasSuffix(path, "/chat/completions"):
return "chat"
case strings.HasSuffix(path, "/completions"):
return "text_completion"
case strings.HasSuffix(path, "/embeddings"):
return "embeddings"
}
return ""
}
// genAIRequest is what a request asked for, as the conventions name it.
func genAIRequest(path string, p *parsedRequest) []attribute.KeyValue {
op := genAIOperation(path)
if op == "" {
return nil
}
kv := []attribute.KeyValue{
attribute.String("gen_ai.operation.name", op),
attribute.String("gen_ai.request.model", p.Model),
}
if p.MaxTokens > 0 {
kv = append(kv, attribute.Int("gen_ai.request.max_tokens", p.MaxTokens))
}
if p.Temperature != nil {
kv = append(kv, attribute.Float64("gen_ai.request.temperature", *p.Temperature))
}
if p.TopP != nil {
kv = append(kv, attribute.Float64("gen_ai.request.top_p", *p.TopP))
}
if p.SessionID != "" {
kv = append(kv, attribute.String("gen_ai.conversation.id", p.SessionID))
}
return kv
}
// genAIUsage is what an answer cost, in the conventions' names: the model
// that answered, and its tokens, cached ones apart.
func genAIUsage(path, served string, u *proxy.Usage) []attribute.KeyValue {
if genAIOperation(path) == "" {
return nil
}
var kv []attribute.KeyValue
if served != "" {
kv = append(kv, attribute.String("gen_ai.response.model", served))
}
if u != nil {
kv = append(kv,
attribute.Int("gen_ai.usage.input_tokens", u.PromptTokens),
attribute.Int("gen_ai.usage.output_tokens", u.CompletionTokens))
if u.CachedTokens > 0 {
kv = append(kv, attribute.Int("gen_ai.usage.cache_read.input_tokens", u.CachedTokens))
}
}
return kv
}
// genAIProvider names who answered an attempt: the engine serving a replica
// (vllm, llamacpp, …), or the outside provider's, for an external model.
func genAIProvider(engine string) string {
if engine == "" {
return "llmgw"
}
return engine
}
+60
View File
@@ -1,6 +1,7 @@
package router
import (
"strings"
"testing"
"go.opentelemetry.io/otel"
@@ -81,3 +82,62 @@ func TestWithoutTracingTheEngineGetsNoTraceparent(t *testing.T) {
t.Errorf("traceparent %q sent with tracing off", tp)
}
}
func TestTracesSpeakTheGenAIConventionsAndHoldNoText(t *testing.T) {
rec := tracetest.NewSpanRecorder()
prevTP := otel.GetTracerProvider()
otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)))
t.Cleanup(func() { otel.SetTracerProvider(prevTP) })
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
res := post(t, rt.Handler(), map[string]any{"model": "m", "max_tokens": 64, "temperature": 0.2, "top_p": 0.9,
"messages": []any{map[string]any{"role": "user", "content": "the secret word is aubergine"}}}, nil)
if res.Code != 200 {
t.Fatalf("status %d: %s", res.Code, res.Body.String())
}
got := map[string]map[string]string{}
for _, s := range rec.Ended() {
attrs := map[string]string{}
for _, a := range s.Attributes() {
attrs[string(a.Key)] = a.Value.Emit()
// Neither the prompt nor the reply ("hi"), in any attribute.
if v := a.Value.Emit(); strings.Contains(v, "aubergine") || v == "hi" {
t.Errorf("span %s carries text in %s", s.Name(), a.Key)
}
}
got[s.Name()] = attrs
}
root := got["inference /v1/chat/completions"]
for k, want := range map[string]string{
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "m",
"gen_ai.request.max_tokens": "64",
"gen_ai.request.temperature": "0.2",
"gen_ai.request.top_p": "0.9",
"gen_ai.response.model": "m",
"gen_ai.usage.input_tokens": "4",
"gen_ai.usage.output_tokens": "2",
} {
if root[k] != want {
t.Errorf("request span %s = %q, want %q", k, root[k], want)
}
}
up := got["upstream"]
if up["gen_ai.operation.name"] != "chat" || up["gen_ai.provider.name"] == "" || up["gen_ai.usage.output_tokens"] != "2" {
t.Errorf("upstream span %v", up)
}
}
func TestEveryInferencePathHasItsGenAIOperation(t *testing.T) {
for path, want := range map[string]string{
"/v1/chat/completions": "chat",
"/v1/completions": "text_completion",
"/v1/embeddings": "embeddings",
"/v1/rerank": "",
} {
if got := genAIOperation(path); got != want {
t.Errorf("%s: %q, want %q", path, got, want)
}
}
}