diff --git a/CHANGELOG.md b/CHANGELOG.md index 37b2c35..dff9d90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,18 @@ and a figure the driver does not give is left out rather than shown as zero. each card and whose quota it counts against, for joining the GPU series to models and tenants. +### The chat page says whether a GPU or a CPU answered + +A model can have replicas on a card and on CPUs at once, and the two answer +at very different speeds; the chat page gave no way to tell which one a reply +came from. The picker now says where each model runs — `GPU`, `CPU` or +`GPU + CPU` — and the line under each reply names what answered it: `GPU +6000-24g` or `CPU`, beside its tokens a second. + +Every answer through the gateway carries the same as `x-llmgw-hardware`: +`cpu`, or the GPU class. An endpoint configured by hand, with no pool, sends +none. + ### A card shared with something else is judged every heartbeat A card with another process on it when its agent started was kept out of diff --git a/internal/router/chat.go b/internal/router/chat.go index 88c641e..8b344ef 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -255,6 +255,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel { } seen := map[string]struct{}{} var out []web.ChatModel + // What each model's replicas compute on, across all of them: one model + // can run on a card and on CPUs at once. + hardware := map[string][]string{} + for _, s := range r.reg.Snapshots("") { + hw := hardwareOf(s.Pool) + for _, m := range s.Models { + name, _ := r.resolve(m) + if hw != "" && !slices.Contains(hardware[name], hw) { + hardware[name] = append(hardware[name], hw) + } + } + } for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { // Resolved rather than taken as sent: an alias and the name it @@ -265,7 +277,7 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel { } seen[name] = struct{}{} if r.permits(principal, name) { - out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()}) + out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated(), Hardware: hardware[name]}) } } } diff --git a/internal/router/hardware_test.go b/internal/router/hardware_test.go new file mode 100644 index 0000000..90c17af --- /dev/null +++ b/internal/router/hardware_test.go @@ -0,0 +1,50 @@ +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" +) + +func TestAReplySaysWhatHardwareAnsweredIt(t *testing.T) { + // qwen2.5-coder-14b had one replica on cuda01's card and one on a CPU + // node, and the chat page could not tell which had answered: the same + // model, ten times faster or slower, for no visible reason. + for pool, want := range map[string]string{ + "none/6000-24g": "6000-24g", + "none/cpu": "cpu", + "": "", + } { + e := newFakeEngine(t, "m") + rt := newTestRouter(t, baseConfig()) + rt.SetEndpoints(t.Context(), []endpoint.Info{ + {ID: "r", URL: e.srv.URL, Models: []string{"m"}, Pool: pool, Weight: 1}, + }) + waitForEndpoint(t, rt) + + rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%q: status %d: %s", pool, rec.Code, rec.Body.String()) + } + if got := rec.Header().Get(hardwareHeader); got != want { + t.Errorf("pool %q: hardware %q, want %q", pool, got, want) + } + } +} + +func TestThePickerSaysWhereEachModelRuns(t *testing.T) { + gpu, cpu := newFakeEngine(t, "qwen2.5-coder-7b-instruct"), newFakeEngine(t, "qwen2.5-coder-7b-instruct") + rt, user := chatRouter(t, gpu.srv.URL) + rt.SetEndpoints(t.Context(), []endpoint.Info{ + {ID: "g", URL: gpu.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, Pool: "none/6000-24g", Weight: 1}, + {ID: "c", URL: cpu.srv.URL, Models: []string{"coder"}, Pool: "none/cpu", Weight: 1}, + }) + waitForEndpoint(t, rt) + + got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user) + if len(got) != 1 || got[0].Runs() != "GPU + CPU" { + t.Fatalf("picker %+v; want one model running on GPU + CPU, its alias's replica included", got) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index b4a6f0d..bfa0503 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -749,6 +749,16 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa if a, ok := r.adapter(req.Model); ok && a.base == model { body = withAdapter(body, model, a.index, a.scale) } + // What computes the answer, said before the proxy writes the headers. + // A model can have replicas on cards and on CPUs at once, and the two + // differ tenfold in speed: a reply that came slowly is otherwise + // indistinguishable from a model that is slow everywhere. Set again on + // a retry, which only happens before anything was written. + if hw := hardwareOf(dec.Endpoint.Pool); hw != "" { + w.Header().Set(hardwareHeader, hw) + } else { + w.Header().Del(hardwareHeader) + } out, perr := r.px.Do(uctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, APIKey: dec.Endpoint.APIKey, @@ -791,6 +801,21 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa return last } +// hardwareHeader says what the replica that answered computes on: "cpu", or +// the GPU class, such as "6000-24g". +const hardwareHeader = "x-llmgw-hardware" + +// hardwareOf reads it from a pool, which placement names fabric/class — +// "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint +// configured by hand, which has no pool and says nothing about its hardware. +func hardwareOf(pool string) string { + i := strings.LastIndexByte(pool, '/') + if i < 0 || i == len(pool)-1 { + return "" + } + return pool[i+1:] +} + func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot { return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id }) } diff --git a/internal/web/chat.go b/internal/web/chat.go index 932495a..a401bbb 100644 --- a/internal/web/chat.go +++ b/internal/web/chat.go @@ -18,6 +18,33 @@ type ChatModel struct { // offered anyway: the first message is what asks, so hiding it until it // runs meant it could never be started from here. OnDemand bool + // Hardware is what the model's running replicas compute on: "cpu", or a + // GPU class. Several when it runs on both, and empty when nothing runs + // or nothing said. + Hardware []string +} + +// Runs names the hardware for the picker: "GPU", "CPU", or "GPU + CPU" for a +// model with replicas on both, where which one answers is the router's choice +// and each reply says which it was. +func (m ChatModel) Runs() string { + var gpu, cpu bool + for _, h := range m.Hardware { + if h == "cpu" { + cpu = true + } else { + gpu = true + } + } + switch { + case gpu && cpu: + return "GPU + CPU" + case gpu: + return "GPU" + case cpu: + return "CPU" + } + return "" } // ChatContext is one thing the chat page can spend against: a project, or the diff --git a/internal/web/chat_test.go b/internal/web/chat_test.go index 63094af..c76c803 100644 --- a/internal/web/chat_test.go +++ b/internal/web/chat_test.go @@ -185,3 +185,19 @@ func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) { t.Errorf("the reply is written as HTML other than from /chat/render (%d assignments)", n) } } + +func TestChatModelRunsNamesTheHardware(t *testing.T) { + for _, c := range []struct { + hw []string + want string + }{ + {nil, ""}, + {[]string{"cpu"}, "CPU"}, + {[]string{"6000-24g"}, "GPU"}, + {[]string{"cpu", "6000-24g"}, "GPU + CPU"}, + } { + if got := (ChatModel{Hardware: c.hw}).Runs(); got != c.want { + t.Errorf("%v: %q, want %q", c.hw, got, c.want) + } + } +} diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index f82b6f5..c957beb 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -29,7 +29,7 @@ {{if .Models}}
@@ -301,10 +301,13 @@ access to it. let busy = null; // A line under a reply: how long it took and how fast it came. - function stats(out, tokens, t0, tFirst, tEnd, stopped) { + function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) { const p = document.createElement("p"); p.className = "note chat-stats"; const parts = []; + // What answered: a model with replicas on a card and on CPUs is served + // by either, and the speed below means nothing without knowing which. + if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw); if (tokens) parts.push(tokens + " tokens"); if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s"); if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s"); @@ -428,7 +431,7 @@ access to it. note.textContent = "Cut off at the length limit — ask it to continue."; out.after(note); } - stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped); + stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware")); // Once, at the end. Re-rendering every chunk would try to typeset half // an expression on each token and flicker its way through the reply. await renderReply(out, reply);