Say on the chat page whether a GPU or a CPU answered
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m54s

A model with replicas on a card and on CPUs answered at either speed with no
way to tell which. The picker names where each model runs, each reply names
the hardware that served it, and every answer carries x-llmgw-hardware.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 18:57:50 +02:00
1 parent 477d0bf19d
commit 1b89d5cdea
7 files changed
+149 -4

No files matched your search

+12
View File
@@ -75,6 +75,18 @@ and a figure the driver does not give is left out rather than shown as zero.
each card and whose quota it counts against, for joining the GPU series to
models and tenants.
### The chat page says whether a GPU or a CPU answered
A model can have replicas on a card and on CPUs at once, and the two answer
at very different speeds; the chat page gave no way to tell which one a reply
came from. The picker now says where each model runs — `GPU`, `CPU` or
`GPU + CPU` — and the line under each reply names what answered it: `GPU
6000-24g` or `CPU`, beside its tokens a second.
Every answer through the gateway carries the same as `x-llmgw-hardware`:
`cpu`, or the GPU class. An endpoint configured by hand, with no pool, sends
none.
### A card shared with something else is judged every heartbeat
A card with another process on it when its agent started was kept out of
+13 -1
View File
@@ -255,6 +255,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
}
seen := map[string]struct{}{}
var out []web.ChatModel
// What each model's replicas compute on, across all of them: one model
// can run on a card and on CPUs at once.
hardware := map[string][]string{}
for _, s := range r.reg.Snapshots("") {
hw := hardwareOf(s.Pool)
for _, m := range s.Models {
name, _ := r.resolve(m)
if hw != "" && !slices.Contains(hardware[name], hw) {
hardware[name] = append(hardware[name], hw)
}
}
}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
// Resolved rather than taken as sent: an alias and the name it
@@ -265,7 +277,7 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
}
seen[name] = struct{}{}
if r.permits(principal, name) {
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()})
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated(), Hardware: hardware[name]})
}
}
}
+50
View File
@@ -0,0 +1,50 @@
package router
import (
"net/http"
"net/http/httptest"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func TestAReplySaysWhatHardwareAnsweredIt(t *testing.T) {
// qwen2.5-coder-14b had one replica on cuda01's card and one on a CPU
// node, and the chat page could not tell which had answered: the same
// model, ten times faster or slower, for no visible reason.
for pool, want := range map[string]string{
"none/6000-24g": "6000-24g",
"none/cpu": "cpu",
"": "",
} {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig())
rt.SetEndpoints(t.Context(), []endpoint.Info{
{ID: "r", URL: e.srv.URL, Models: []string{"m"}, Pool: pool, Weight: 1},
})
waitForEndpoint(t, rt)
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%q: status %d: %s", pool, rec.Code, rec.Body.String())
}
if got := rec.Header().Get(hardwareHeader); got != want {
t.Errorf("pool %q: hardware %q, want %q", pool, got, want)
}
}
}
func TestThePickerSaysWhereEachModelRuns(t *testing.T) {
gpu, cpu := newFakeEngine(t, "qwen2.5-coder-7b-instruct"), newFakeEngine(t, "qwen2.5-coder-7b-instruct")
rt, user := chatRouter(t, gpu.srv.URL)
rt.SetEndpoints(t.Context(), []endpoint.Info{
{ID: "g", URL: gpu.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, Pool: "none/6000-24g", Weight: 1},
{ID: "c", URL: cpu.srv.URL, Models: []string{"coder"}, Pool: "none/cpu", Weight: 1},
})
waitForEndpoint(t, rt)
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
if len(got) != 1 || got[0].Runs() != "GPU + CPU" {
t.Fatalf("picker %+v; want one model running on GPU + CPU, its alias's replica included", got)
}
}
+25
View File
@@ -749,6 +749,16 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
if a, ok := r.adapter(req.Model); ok && a.base == model {
body = withAdapter(body, model, a.index, a.scale)
}
// What computes the answer, said before the proxy writes the headers.
// A model can have replicas on cards and on CPUs at once, and the two
// differ tenfold in speed: a reply that came slowly is otherwise
// indistinguishable from a model that is slow everywhere. Set again on
// a retry, which only happens before anything was written.
if hw := hardwareOf(dec.Endpoint.Pool); hw != "" {
w.Header().Set(hardwareHeader, hw)
} else {
w.Header().Del(hardwareHeader)
}
out, perr := r.px.Do(uctx, w, proxy.Request{
URL: dec.Endpoint.URL + req.Path,
APIKey: dec.Endpoint.APIKey,
@@ -791,6 +801,21 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
return last
}
// hardwareHeader says what the replica that answered computes on: "cpu", or
// the GPU class, such as "6000-24g".
const hardwareHeader = "x-llmgw-hardware"
// hardwareOf reads it from a pool, which placement names fabric/class —
// "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint
// configured by hand, which has no pool and says nothing about its hardware.
func hardwareOf(pool string) string {
i := strings.LastIndexByte(pool, '/')
if i < 0 || i == len(pool)-1 {
return ""
}
return pool[i+1:]
}
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
}
+27
View File
@@ -18,6 +18,33 @@ type ChatModel struct {
// offered anyway: the first message is what asks, so hiding it until it
// runs meant it could never be started from here.
OnDemand bool
// Hardware is what the model's running replicas compute on: "cpu", or a
// GPU class. Several when it runs on both, and empty when nothing runs
// or nothing said.
Hardware []string
}
// Runs names the hardware for the picker: "GPU", "CPU", or "GPU + CPU" for a
// model with replicas on both, where which one answers is the router's choice
// and each reply says which it was.
func (m ChatModel) Runs() string {
var gpu, cpu bool
for _, h := range m.Hardware {
if h == "cpu" {
cpu = true
} else {
gpu = true
}
}
switch {
case gpu && cpu:
return "GPU + CPU"
case gpu:
return "GPU"
case cpu:
return "CPU"
}
return ""
}
// ChatContext is one thing the chat page can spend against: a project, or the
+16
View File
@@ -185,3 +185,19 @@ func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) {
t.Errorf("the reply is written as HTML other than from /chat/render (%d assignments)", n)
}
}
func TestChatModelRunsNamesTheHardware(t *testing.T) {
for _, c := range []struct {
hw []string
want string
}{
{nil, ""},
{[]string{"cpu"}, "CPU"},
{[]string{"6000-24g"}, "GPU"},
{[]string{"cpu", "6000-24g"}, "GPU + CPU"},
} {
if got := (ChatModel{Hardware: c.hw}).Runs(); got != c.want {
t.Errorf("%v: %q, want %q", c.hw, got, c.want)
}
}
}
+6 -3
View File
@@ -29,7 +29,7 @@
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
@@ -301,10 +301,13 @@ access to it.</p>
let busy = null;
// A line under a reply: how long it took and how fast it came.
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
const p = document.createElement("p");
p.className = "note chat-stats";
const parts = [];
// What answered: a model with replicas on a card and on CPUs is served
// by either, and the speed below means nothing without knowing which.
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
if (tokens) parts.push(tokens + " tokens");
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
@@ -428,7 +431,7 @@ access to it.</p>
note.textContent = "Cut off at the length limit — ask it to continue.";
out.after(note);
}
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
// Once, at the end. Re-rendering every chunk would try to typeset half
// an expression on each token and flicker its way through the reply.
await renderReply(out, reply);