Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b89d5cdea |
No files matched your search
@@ -75,6 +75,18 @@ and a figure the driver does not give is left out rather than shown as zero.
|
||||
each card and whose quota it counts against, for joining the GPU series to
|
||||
models and tenants.
|
||||
|
||||
### The chat page says whether a GPU or a CPU answered
|
||||
|
||||
A model can have replicas on a card and on CPUs at once, and the two answer
|
||||
at very different speeds; the chat page gave no way to tell which one a reply
|
||||
came from. The picker now says where each model runs — `GPU`, `CPU` or
|
||||
`GPU + CPU` — and the line under each reply names what answered it: `GPU
|
||||
6000-24g` or `CPU`, beside its tokens a second.
|
||||
|
||||
Every answer through the gateway carries the same as `x-llmgw-hardware`:
|
||||
`cpu`, or the GPU class. An endpoint configured by hand, with no pool, sends
|
||||
none.
|
||||
|
||||
### A card shared with something else is judged every heartbeat
|
||||
|
||||
A card with another process on it when its agent started was kept out of
|
||||
|
||||
+13
-1
@@ -255,6 +255,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var out []web.ChatModel
|
||||
// What each model's replicas compute on, across all of them: one model
|
||||
// can run on a card and on CPUs at once.
|
||||
hardware := map[string][]string{}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
hw := hardwareOf(s.Pool)
|
||||
for _, m := range s.Models {
|
||||
name, _ := r.resolve(m)
|
||||
if hw != "" && !slices.Contains(hardware[name], hw) {
|
||||
hardware[name] = append(hardware[name], hw)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
for _, m := range s.Models {
|
||||
// Resolved rather than taken as sent: an alias and the name it
|
||||
@@ -265,7 +277,7 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
if r.permits(principal, name) {
|
||||
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()})
|
||||
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated(), Hardware: hardware[name]})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
func TestAReplySaysWhatHardwareAnsweredIt(t *testing.T) {
|
||||
// qwen2.5-coder-14b had one replica on cuda01's card and one on a CPU
|
||||
// node, and the chat page could not tell which had answered: the same
|
||||
// model, ten times faster or slower, for no visible reason.
|
||||
for pool, want := range map[string]string{
|
||||
"none/6000-24g": "6000-24g",
|
||||
"none/cpu": "cpu",
|
||||
"": "",
|
||||
} {
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig())
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "r", URL: e.srv.URL, Models: []string{"m"}, Pool: pool, Weight: 1},
|
||||
})
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%q: status %d: %s", pool, rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get(hardwareHeader); got != want {
|
||||
t.Errorf("pool %q: hardware %q, want %q", pool, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestThePickerSaysWhereEachModelRuns(t *testing.T) {
|
||||
gpu, cpu := newFakeEngine(t, "qwen2.5-coder-7b-instruct"), newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
||||
rt, user := chatRouter(t, gpu.srv.URL)
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "g", URL: gpu.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, Pool: "none/6000-24g", Weight: 1},
|
||||
{ID: "c", URL: cpu.srv.URL, Models: []string{"coder"}, Pool: "none/cpu", Weight: 1},
|
||||
})
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
|
||||
if len(got) != 1 || got[0].Runs() != "GPU + CPU" {
|
||||
t.Fatalf("picker %+v; want one model running on GPU + CPU, its alias's replica included", got)
|
||||
}
|
||||
}
|
||||
@@ -749,6 +749,16 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
if a, ok := r.adapter(req.Model); ok && a.base == model {
|
||||
body = withAdapter(body, model, a.index, a.scale)
|
||||
}
|
||||
// What computes the answer, said before the proxy writes the headers.
|
||||
// A model can have replicas on cards and on CPUs at once, and the two
|
||||
// differ tenfold in speed: a reply that came slowly is otherwise
|
||||
// indistinguishable from a model that is slow everywhere. Set again on
|
||||
// a retry, which only happens before anything was written.
|
||||
if hw := hardwareOf(dec.Endpoint.Pool); hw != "" {
|
||||
w.Header().Set(hardwareHeader, hw)
|
||||
} else {
|
||||
w.Header().Del(hardwareHeader)
|
||||
}
|
||||
out, perr := r.px.Do(uctx, w, proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
@@ -791,6 +801,21 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
return last
|
||||
}
|
||||
|
||||
// hardwareHeader says what the replica that answered computes on: "cpu", or
|
||||
// the GPU class, such as "6000-24g".
|
||||
const hardwareHeader = "x-llmgw-hardware"
|
||||
|
||||
// hardwareOf reads it from a pool, which placement names fabric/class —
|
||||
// "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint
|
||||
// configured by hand, which has no pool and says nothing about its hardware.
|
||||
func hardwareOf(pool string) string {
|
||||
i := strings.LastIndexByte(pool, '/')
|
||||
if i < 0 || i == len(pool)-1 {
|
||||
return ""
|
||||
}
|
||||
return pool[i+1:]
|
||||
}
|
||||
|
||||
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
||||
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
||||
}
|
||||
|
||||
@@ -18,6 +18,33 @@ type ChatModel struct {
|
||||
// offered anyway: the first message is what asks, so hiding it until it
|
||||
// runs meant it could never be started from here.
|
||||
OnDemand bool
|
||||
// Hardware is what the model's running replicas compute on: "cpu", or a
|
||||
// GPU class. Several when it runs on both, and empty when nothing runs
|
||||
// or nothing said.
|
||||
Hardware []string
|
||||
}
|
||||
|
||||
// Runs names the hardware for the picker: "GPU", "CPU", or "GPU + CPU" for a
|
||||
// model with replicas on both, where which one answers is the router's choice
|
||||
// and each reply says which it was.
|
||||
func (m ChatModel) Runs() string {
|
||||
var gpu, cpu bool
|
||||
for _, h := range m.Hardware {
|
||||
if h == "cpu" {
|
||||
cpu = true
|
||||
} else {
|
||||
gpu = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case gpu && cpu:
|
||||
return "GPU + CPU"
|
||||
case gpu:
|
||||
return "GPU"
|
||||
case cpu:
|
||||
return "CPU"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ChatContext is one thing the chat page can spend against: a project, or the
|
||||
|
||||
@@ -185,3 +185,19 @@ func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) {
|
||||
t.Errorf("the reply is written as HTML other than from /chat/render (%d assignments)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatModelRunsNamesTheHardware(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
hw []string
|
||||
want string
|
||||
}{
|
||||
{nil, ""},
|
||||
{[]string{"cpu"}, "CPU"},
|
||||
{[]string{"6000-24g"}, "GPU"},
|
||||
{[]string{"cpu", "6000-24g"}, "GPU + CPU"},
|
||||
} {
|
||||
if got := (ChatModel{Hardware: c.hw}).Runs(); got != c.want {
|
||||
t.Errorf("%v: %q, want %q", c.hw, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,7 @@
|
||||
{{if .Models}}
|
||||
<div class="chat-bar">
|
||||
<label class="inline-label">Model
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
</label>
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
@@ -301,10 +301,13 @@ access to it.</p>
|
||||
let busy = null;
|
||||
|
||||
// A line under a reply: how long it took and how fast it came.
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
|
||||
const p = document.createElement("p");
|
||||
p.className = "note chat-stats";
|
||||
const parts = [];
|
||||
// What answered: a model with replicas on a card and on CPUs is served
|
||||
// by either, and the speed below means nothing without knowing which.
|
||||
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
|
||||
if (tokens) parts.push(tokens + " tokens");
|
||||
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
|
||||
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
|
||||
@@ -428,7 +431,7 @@ access to it.</p>
|
||||
note.textContent = "Cut off at the length limit — ask it to continue.";
|
||||
out.after(note);
|
||||
}
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
|
||||
// Once, at the end. Re-rendering every chunk would try to typeset half
|
||||
// an expression on each token and flicker its way through the reply.
|
||||
await renderReply(out, reply);
|
||||
|
||||
Reference in new issue
Block a user