Say on the chat page whether a GPU or a CPU answered
A model with replicas on a card and on CPUs answered at either speed with no way to tell which. The picker names where each model runs, each reply names the hardware that served it, and every answer carries x-llmgw-hardware. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
477d0bf19d
commit
1b89d5cdea
7 files changed
+149
-4
No files matched your search
@@ -75,6 +75,18 @@ and a figure the driver does not give is left out rather than shown as zero.
|
||||
each card and whose quota it counts against, for joining the GPU series to
|
||||
models and tenants.
|
||||
|
||||
### The chat page says whether a GPU or a CPU answered
|
||||
|
||||
A model can have replicas on a card and on CPUs at once, and the two answer
|
||||
at very different speeds; the chat page gave no way to tell which one a reply
|
||||
came from. The picker now says where each model runs — `GPU`, `CPU` or
|
||||
`GPU + CPU` — and the line under each reply names what answered it: `GPU
|
||||
6000-24g` or `CPU`, beside its tokens a second.
|
||||
|
||||
Every answer through the gateway carries the same as `x-llmgw-hardware`:
|
||||
`cpu`, or the GPU class. An endpoint configured by hand, with no pool, sends
|
||||
none.
|
||||
|
||||
### A card shared with something else is judged every heartbeat
|
||||
|
||||
A card with another process on it when its agent started was kept out of
|
||||
|
||||
+13
-1
@@ -255,6 +255,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var out []web.ChatModel
|
||||
// What each model's replicas compute on, across all of them: one model
|
||||
// can run on a card and on CPUs at once.
|
||||
hardware := map[string][]string{}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
hw := hardwareOf(s.Pool)
|
||||
for _, m := range s.Models {
|
||||
name, _ := r.resolve(m)
|
||||
if hw != "" && !slices.Contains(hardware[name], hw) {
|
||||
hardware[name] = append(hardware[name], hw)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
for _, m := range s.Models {
|
||||
// Resolved rather than taken as sent: an alias and the name it
|
||||
@@ -265,7 +277,7 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
if r.permits(principal, name) {
|
||||
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()})
|
||||
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated(), Hardware: hardware[name]})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
func TestAReplySaysWhatHardwareAnsweredIt(t *testing.T) {
|
||||
// qwen2.5-coder-14b had one replica on cuda01's card and one on a CPU
|
||||
// node, and the chat page could not tell which had answered: the same
|
||||
// model, ten times faster or slower, for no visible reason.
|
||||
for pool, want := range map[string]string{
|
||||
"none/6000-24g": "6000-24g",
|
||||
"none/cpu": "cpu",
|
||||
"": "",
|
||||
} {
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig())
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "r", URL: e.srv.URL, Models: []string{"m"}, Pool: pool, Weight: 1},
|
||||
})
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%q: status %d: %s", pool, rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get(hardwareHeader); got != want {
|
||||
t.Errorf("pool %q: hardware %q, want %q", pool, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestThePickerSaysWhereEachModelRuns(t *testing.T) {
|
||||
gpu, cpu := newFakeEngine(t, "qwen2.5-coder-7b-instruct"), newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
||||
rt, user := chatRouter(t, gpu.srv.URL)
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "g", URL: gpu.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, Pool: "none/6000-24g", Weight: 1},
|
||||
{ID: "c", URL: cpu.srv.URL, Models: []string{"coder"}, Pool: "none/cpu", Weight: 1},
|
||||
})
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
|
||||
if len(got) != 1 || got[0].Runs() != "GPU + CPU" {
|
||||
t.Fatalf("picker %+v; want one model running on GPU + CPU, its alias's replica included", got)
|
||||
}
|
||||
}
|
||||
@@ -749,6 +749,16 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
if a, ok := r.adapter(req.Model); ok && a.base == model {
|
||||
body = withAdapter(body, model, a.index, a.scale)
|
||||
}
|
||||
// What computes the answer, said before the proxy writes the headers.
|
||||
// A model can have replicas on cards and on CPUs at once, and the two
|
||||
// differ tenfold in speed: a reply that came slowly is otherwise
|
||||
// indistinguishable from a model that is slow everywhere. Set again on
|
||||
// a retry, which only happens before anything was written.
|
||||
if hw := hardwareOf(dec.Endpoint.Pool); hw != "" {
|
||||
w.Header().Set(hardwareHeader, hw)
|
||||
} else {
|
||||
w.Header().Del(hardwareHeader)
|
||||
}
|
||||
out, perr := r.px.Do(uctx, w, proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
@@ -791,6 +801,21 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
return last
|
||||
}
|
||||
|
||||
// hardwareHeader says what the replica that answered computes on: "cpu", or
|
||||
// the GPU class, such as "6000-24g".
|
||||
const hardwareHeader = "x-llmgw-hardware"
|
||||
|
||||
// hardwareOf reads it from a pool, which placement names fabric/class —
|
||||
// "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint
|
||||
// configured by hand, which has no pool and says nothing about its hardware.
|
||||
func hardwareOf(pool string) string {
|
||||
i := strings.LastIndexByte(pool, '/')
|
||||
if i < 0 || i == len(pool)-1 {
|
||||
return ""
|
||||
}
|
||||
return pool[i+1:]
|
||||
}
|
||||
|
||||
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
||||
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
||||
}
|
||||
|
||||
@@ -18,6 +18,33 @@ type ChatModel struct {
|
||||
// offered anyway: the first message is what asks, so hiding it until it
|
||||
// runs meant it could never be started from here.
|
||||
OnDemand bool
|
||||
// Hardware is what the model's running replicas compute on: "cpu", or a
|
||||
// GPU class. Several when it runs on both, and empty when nothing runs
|
||||
// or nothing said.
|
||||
Hardware []string
|
||||
}
|
||||
|
||||
// Runs names the hardware for the picker: "GPU", "CPU", or "GPU + CPU" for a
|
||||
// model with replicas on both, where which one answers is the router's choice
|
||||
// and each reply says which it was.
|
||||
func (m ChatModel) Runs() string {
|
||||
var gpu, cpu bool
|
||||
for _, h := range m.Hardware {
|
||||
if h == "cpu" {
|
||||
cpu = true
|
||||
} else {
|
||||
gpu = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case gpu && cpu:
|
||||
return "GPU + CPU"
|
||||
case gpu:
|
||||
return "GPU"
|
||||
case cpu:
|
||||
return "CPU"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ChatContext is one thing the chat page can spend against: a project, or the
|
||||
|
||||
@@ -185,3 +185,19 @@ func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) {
|
||||
t.Errorf("the reply is written as HTML other than from /chat/render (%d assignments)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatModelRunsNamesTheHardware(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
hw []string
|
||||
want string
|
||||
}{
|
||||
{nil, ""},
|
||||
{[]string{"cpu"}, "CPU"},
|
||||
{[]string{"6000-24g"}, "GPU"},
|
||||
{[]string{"cpu", "6000-24g"}, "GPU + CPU"},
|
||||
} {
|
||||
if got := (ChatModel{Hardware: c.hw}).Runs(); got != c.want {
|
||||
t.Errorf("%v: %q, want %q", c.hw, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,7 @@
|
||||
{{if .Models}}
|
||||
<div class="chat-bar">
|
||||
<label class="inline-label">Model
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
</label>
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
@@ -301,10 +301,13 @@ access to it.</p>
|
||||
let busy = null;
|
||||
|
||||
// A line under a reply: how long it took and how fast it came.
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
|
||||
const p = document.createElement("p");
|
||||
p.className = "note chat-stats";
|
||||
const parts = [];
|
||||
// What answered: a model with replicas on a card and on CPUs is served
|
||||
// by either, and the speed below means nothing without knowing which.
|
||||
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
|
||||
if (tokens) parts.push(tokens + " tokens");
|
||||
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
|
||||
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
|
||||
@@ -428,7 +431,7 @@ access to it.</p>
|
||||
note.textContent = "Cut off at the length limit — ask it to continue.";
|
||||
out.after(note);
|
||||
}
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
|
||||
// Once, at the end. Re-rendering every chunk would try to typeset half
|
||||
// an expression on each token and flicker its way through the reply.
|
||||
await renderReply(out, reply);
|
||||
|
||||
Reference in new issue
Block a user