Compare commits

...
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 1b89d5cdea Say on the chat page whether a GPU or a CPU answered
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m54s
A model with replicas on a card and on CPUs answered at either speed with no
way to tell which. The picker names where each model runs, each reply names
the hardware that served it, and every answer carries x-llmgw-hardware.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 18:57:50 +02:00
Ilker ManapandClaude Opus 5.5 477d0bf19d Judge a card shared with another process every heartbeat, not once
Release / build (push) Successful in 1m14s
Release / notes (push) Successful in 4s
CI / test (push) Successful in 3m50s
A card with something else on it when the agent started stayed out of
placement after that process let go of it, until the agent restarted. The
agent now sends what processes other than its own engines hold on each card,
and the control plane replaces the registration verdict with it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 18:46:23 +02:00
17 changed files with 461 additions and 22 deletions

No files matched your search

+27
View File
@@ -75,6 +75,33 @@ and a figure the driver does not give is left out rather than shown as zero.
each card and whose quota it counts against, for joining the GPU series to
models and tenants.
### The chat page says whether a GPU or a CPU answered
A model can have replicas on a card and on CPUs at once, and the two answer
at very different speeds; the chat page gave no way to tell which one a reply
came from. The picker now says where each model runs — `GPU`, `CPU` or
`GPU + CPU` — and the line under each reply names what answered it: `GPU
6000-24g` or `CPU`, beside its tokens a second.
Every answer through the gateway carries the same as `x-llmgw-hardware`:
`cpu`, or the GPU class. An endpoint configured by hand, with no pool, sends
none.
### A card shared with something else is judged every heartbeat
A card with another process on it when its agent started was kept out of
placement, and stayed out — "5014 MiB already allocated by another process" —
after that process let go of it, until the agent was restarted. Found on a
node shared with ComfyUI. The other way round, a tenant arriving after
registration was never seen, and a card the agent's own engine held when the
agent restarted read as someone else's.
The agent now reads every card's memory per process on each heartbeat,
leaves out what its own engines hold, and sends the rest; the control plane
keeps a card out of placement on that reading, and lets it back in when it
falls under 512 MiB. `llmgw_gpu_memory_used_bytes` reads the same figure. An
agent too old to send it keeps the reading from registration.
### A GPU placement no longer runs on the CPU unnoticed
llama.cpp placed on a GPU, with its CUDA backend unable to load (cuBLAS not
+21 -5
View File
@@ -2336,9 +2336,17 @@ type Heartbeat struct {
GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
// cpu_count as in Register, for a VM given more CPUs while it runs.
CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
// gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
// this agent's shards hold on each card, read afresh every heartbeat. What
// keeps a card out of placement for another tenant is decided from this
// rather than from registration: a card that had something else on it when
// the agent started is not taken forever, and one that gains a tenant later
// is not handed out. Empty from an agent that does not measure it, or from
// one that could not this time.
GpuForeignMemoryMb []int32 `protobuf:"varint,19,rep,packed,name=gpu_foreign_memory_mb,json=gpuForeignMemoryMb,proto3" json:"gpu_foreign_memory_mb,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Heartbeat) Reset() {
@@ -2497,6 +2505,13 @@ func (x *Heartbeat) GetCpuCount() int32 {
return 0
}
func (x *Heartbeat) GetGpuForeignMemoryMb() []int32 {
if x != nil {
return x.GpuForeignMemoryMb
}
return nil
}
// Provision is the software a node should have: which engines, at which
// versions. Nothing more.
//
@@ -4260,7 +4275,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x06status\x18\x03 \x01(\tR\x06status\x12!\n" +
"\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" +
"\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" +
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xf6\x06\n" +
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xa9\a\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -4281,7 +4296,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" +
"\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" +
"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x1aA\n" +
"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x121\n" +
"\x15gpu_foreign_memory_mb\x18\x13 \x03(\x05R\x12gpuForeignMemoryMb\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
+9
View File
@@ -474,6 +474,15 @@ message Heartbeat {
IMEX imex = 17;
// cpu_count as in Register, for a VM given more CPUs while it runs.
int32 cpu_count = 18;
// gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
// this agent's shards hold on each card, read afresh every heartbeat. What
// keeps a card out of placement for another tenant is decided from this
// rather than from registration: a card that had something else on it when
// the agent started is not taken forever, and one that gains a tenant later
// is not handed out. Empty from an agent that does not measure it, or from
// one that could not this time.
repeated int32 gpu_foreign_memory_mb = 19;
}
// Provision is the software a node should have: which engines, at which
+18 -2
View File
@@ -591,8 +591,24 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
}
a.mu.Unlock()
for _, g := range inv.GPUs {
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
// Measured now, not carried from registration: a card is shared with
// whatever else runs on the machine, and that comes and goes while the
// agent runs. The registration figure stays only for a node where
// nvidia-smi cannot be asked.
if len(inv.GPUs) > 0 {
var shards []int
for _, st := range a.sup.Statuses() {
if st.PID > 0 {
shards = append(shards, st.PID)
}
}
hb.GpuForeignMemoryMb, hb.GpuMemoryUsedMb = a.foreignGPUMemory(context.Background(), shards)
}
if len(hb.GpuMemoryUsedMb) != len(inv.GPUs) {
hb.GpuForeignMemoryMb, hb.GpuMemoryUsedMb = nil, nil
for _, g := range inv.GPUs {
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
}
}
// Rescanned here rather than reused from registration. Disk is not a fact
+27 -5
View File
@@ -106,9 +106,27 @@ func parseSMI(out string) ([]GPU, error) {
return gpus, nil
}
// residentMemoryToleranceMB is how much allocated memory is written off as
// ForeignMemoryToleranceMB is how much allocated memory is written off as
// driver overhead rather than a foreign process.
const residentMemoryToleranceMB = 512
const ForeignMemoryToleranceMB = 512
// foreignMemorySuffix ends the problem a card with another tenant on it is
// kept out of placement for. One phrase, so that the control plane can find
// it again and drop it when the tenant leaves.
const foreignMemorySuffix = " MiB already allocated by another process"
// ForeignMemoryProblem is the problem for mb of memory held by something
// that is not one of this agent's engines, or "" when that is too little to
// count.
func ForeignMemoryProblem(mb int) string {
if mb <= ForeignMemoryToleranceMB {
return ""
}
return fmt.Sprintf("%d%s", mb, foreignMemorySuffix)
}
// IsForeignMemoryProblem reports whether a problem is ForeignMemoryProblem's.
func IsForeignMemoryProblem(p string) bool { return strings.HasSuffix(p, foreignMemorySuffix) }
// assessHealth flags conditions that should keep a GPU out of placement.
func assessHealth(g *GPU) {
@@ -119,8 +137,12 @@ func assessHealth(g *GPU) {
// Memory already in use means something else is on the card. Starting an
// engine there would either fail to allocate or quietly evict the other
// process's throughput.
if g.MemoryUsedMB > residentMemoryToleranceMB {
g.Health.Problems = append(g.Health.Problems,
fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
//
// Only a first reading: this runs before the agent knows which processes
// are its own, so an engine still running from before a restart reads as
// foreign here. Every heartbeat measures it again, without the agent's
// own engines, and the control plane replaces this with that.
if p := ForeignMemoryProblem(g.MemoryUsedMB); p != "" {
g.Health.Problems = append(g.Health.Problems, p)
}
}
+61 -5
View File
@@ -34,13 +34,47 @@ func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
return parseGPUProcesses(out)
}
// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
// is allocated on it, and which processes hold how much of that.
type GPUMemory struct {
UsedMB int
Procs []GPUProcess
}
// GPUProcess is one process holding memory on a card.
type GPUProcess struct {
PID int
UsedMB int
}
// GPUMemoryByProcess reports each GPU's memory, by index, with the processes
// holding it — from the same report as GPUProcesses, so the two never
// disagree about who is on a card.
//
// A node with no nvidia-smi returns nil and no error, as discovery does.
func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) {
if binary == "" {
if _, err := exec.LookPath("nvidia-smi"); err != nil {
return nil, nil
}
binary = "nvidia-smi"
}
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
if err != nil {
return nil, fmt.Errorf("running %s: %w", binary, err)
}
return parseGPUMemory(out)
}
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
// index order, and the processes on it.
// index order, its memory and the processes on it.
type smiLog struct {
GPUs []struct {
Used string `xml:"fb_memory_usage>used"`
Processes []struct {
PID string `xml:"pid"`
Type string `xml:"type"`
PID string `xml:"pid"`
Type string `xml:"type"`
UsedMB string `xml:"used_memory"`
} `xml:"processes>process_info"`
} `xml:"gpu"`
}
@@ -48,17 +82,39 @@ type smiLog struct {
// parseGPUProcesses reads the report. GPUs are listed in index order, which
// is nvidia-smi's own numbering — the one placement assigns by.
func parseGPUProcesses(report []byte) (map[int][]int, error) {
mem, err := parseGPUMemory(report)
if err != nil {
return nil, err
}
out := map[int][]int{}
for i, g := range mem {
for _, p := range g.Procs {
out[i] = append(out[i], p.PID)
}
}
return out, nil
}
// parseGPUMemory reads the report into each GPU's memory and its holders.
func parseGPUMemory(report []byte) ([]GPUMemory, error) {
var log smiLog
if err := xml.Unmarshal(report, &log); err != nil {
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
}
out := map[int][]int{}
out := make([]GPUMemory, len(log.GPUs))
for i, g := range log.GPUs {
out[i].UsedMB = mib(g.Used)
for _, p := range g.Processes {
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
out[i] = append(out[i], pid)
out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
}
}
}
return out, nil
}
// mib reads "4932 MiB"; anything else, "N/A" included, is zero.
func mib(s string) int {
n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB")))
return n
}
+40
View File
@@ -36,3 +36,43 @@ func TestGPUProcessesOfEveryKindAreListedByIndex(t *testing.T) {
t.Error("a report that does not parse was read as no processes")
}
}
func TestGPUMemoryIsReadPerProcess(t *testing.T) {
report := `<?xml version="1.0" ?>
<nvidia_smi_log>
<gpu id="00000000:01:00.0"><product_name>Quadro RTX 6000</product_name>
<fb_memory_usage><total>24576 MiB</total><used>14514 MiB</used><free>10062 MiB</free></fb_memory_usage>
<processes>
<process_info><pid>71315</pid><type>C</type><used_memory>4932 MiB</used_memory></process_info>
<process_info><pid>4100</pid><type>C</type><used_memory>9500 MiB</used_memory></process_info>
</processes>
</gpu>
<gpu id="00000000:02:00.0">
<fb_memory_usage><used>N/A</used></fb_memory_usage>
<processes></processes>
</gpu>
</nvidia_smi_log>`
got, err := parseGPUMemory([]byte(report))
if err != nil {
t.Fatal(err)
}
want := []GPUMemory{
{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
{},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
}
func TestForeignMemoryBelowTheToleranceIsNoProblem(t *testing.T) {
// A CUDA context that has let go of its model still holds a few hundred
// MiB: ComfyUI after /free held 324.
if p := ForeignMemoryProblem(330); p != "" {
t.Errorf("330 MiB is a problem: %q", p)
}
p := ForeignMemoryProblem(5014)
if !IsForeignMemoryProblem(p) || IsForeignMemoryProblem("2 uncorrected ECC errors") {
t.Errorf("%q is not recognised as the foreign memory problem, or ECC is", p)
}
}
+39
View File
@@ -16,6 +16,7 @@ import (
// nvidia-smi and /proc.
var (
gpuProcesses = discovery.GPUProcesses
gpuMemory = discovery.GPUMemoryByProcess
processGroup = pgidOf
// gpuCheckWait is how long a shard has, once it answers, to show up on
// its GPUs: the weights are on them by then, but nvidia-smi can lag.
@@ -75,6 +76,44 @@ func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.
"or run `llmgw doctor` on this node", joinIndices(missing), seen)
}
// foreignGPUMemory is what each GPU holds for anything other than this
// agent's shards, by index, and each GPU's memory in use. Nil when nvidia-smi
// cannot be asked, which is no reading rather than an empty card.
//
// Everything allocated less what the shards' processes hold, rather than the
// sum of the other processes: a process in another container's PID namespace
// holds memory nvidia-smi lists under no PID this node can see, and it is
// still not memory placement can hand out.
func (a *Agent) foreignGPUMemory(ctx context.Context, shards []int) (foreign, used []int32) {
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
if err != nil || mem == nil {
return nil, nil
}
groups := make([]int, len(shards))
for i, pid := range shards {
groups[i] = processGroup(pid)
}
ours := func(p int) bool {
for i, pid := range shards {
if holds([]int{p}, pid, groups[i]) {
return true
}
}
return false
}
for _, g := range mem {
f := g.UsedMB
for _, p := range g.Procs {
if ours(p.PID) {
f -= p.UsedMB
}
}
foreign = append(foreign, int32(max(f, 0)))
used = append(used, int32(g.UsedMB))
}
return foreign, used
}
// holds reports whether any of pids is the shard's process or in its group:
// an engine that computes in a child process is still the shard.
func holds(pids []int, pid, group int) bool {
+33
View File
@@ -3,10 +3,12 @@ package agent
import (
"context"
"log/slog"
"slices"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
@@ -89,3 +91,34 @@ func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) {
t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err)
}
}
func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
// cuda01: ComfyUI and one of our engines on the same card. Only ComfyUI's
// share — and whatever no listed process accounts for — is foreign.
oldMem, oldGroup := gpuMemory, processGroup
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
return []discovery.GPUMemory{
{UsedMB: 15000, Procs: []discovery.GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 512, UsedMB: 9500}}},
{UsedMB: 300},
}, nil
}
processGroup = func(pid int) int {
if pid >= 500 && pid < 600 {
return 500
}
return pid
}
t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
a := &Agent{log: slog.Default()}
foreign, used := a.foreignGPUMemory(context.Background(), []int{500})
if !slices.Equal(foreign, []int32{5500, 300}) || !slices.Equal(used, []int32{15000, 300}) {
t.Errorf("foreign %v, used %v; want [5500 300] and [15000 300]", foreign, used)
}
// No nvidia-smi to ask is no reading, not an empty card.
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
if foreign, _ := a.foreignGPUMemory(context.Background(), nil); foreign != nil {
t.Errorf("no reading came back as %v", foreign)
}
}
@@ -283,3 +283,36 @@ func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
t.Errorf("the agent's refusal was not passed on: %v", err)
}
}
func TestAnotherTenantOnACardIsJudgedEveryHeartbeat(t *testing.T) {
// Found on cuda01: ComfyUI held 5 GB when the agent started, so the card
// was kept out of placement. ComfyUI let go of it and the card stayed
// out, saying "5014 MiB already allocated by another process", until the
// agent was restarted.
s := newTestService()
nodeWith(s, inventory.Node{
ID: "n1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Health: discovery.GPUHealth{Problems: []string{
discovery.ForeignMemoryProblem(5014), "2 uncorrected ECC errors"}}}},
})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", GpuForeignMemoryMb: []int32{330}})
n, _ := s.inv.Get("n1")
if got := n.GPUs[0].Health.Problems; len(got) != 1 || got[0] != "2 uncorrected ECC errors" {
t.Fatalf("problems are %q: the tenant left, and only the ECC errors should remain", got)
}
// And the other way: a tenant arriving after registration.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", GpuForeignMemoryMb: []int32{12000}})
n, _ = s.inv.Get("n1")
if got := n.GPUs[0].Health.Problems; len(got) != 2 || got[1] != discovery.ForeignMemoryProblem(12000) {
t.Fatalf("problems are %q, want the new tenant's 12000 MiB", got)
}
// An agent that sends no reading changes nothing.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", GpuMemoryUsedMb: []int32{100}})
n, _ = s.inv.Get("n1")
if len(n.GPUs[0].Health.Problems) != 2 {
t.Errorf("a heartbeat with no foreign reading cleared the card: %q", n.GPUs[0].Health.Problems)
}
}
+16 -1
View File
@@ -377,11 +377,12 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
// a reconnect.
func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
used := hb.GetGpuMemoryUsedMb()
foreign := hb.GetGpuForeignMemoryMb()
problems := hb.GetNewProblems()
storage := hb.GetStorage()
memMB := int(hb.GetMemoryTotalMb())
cpus := int(hb.GetCpuCount())
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
if len(used) == 0 && len(foreign) == 0 && len(problems) == 0 && len(storage) == 0 &&
!hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
@@ -397,6 +398,20 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
n.GPUs[i].MemoryUsedMB = int(used[i])
}
}
// Another tenant on a card, judged afresh: the reading at registration is
// replaced, both ways. Without this a card that had something on it when
// its agent started stayed out of placement after it left, and said so,
// until the agent was restarted. Only a reading for every card replaces
// it; an agent that sends none, or could not measure, leaves it standing.
if len(foreign) == len(n.GPUs) {
for i := range n.GPUs {
ps := slices.DeleteFunc(slices.Clone(n.GPUs[i].Health.Problems), discovery.IsForeignMemoryProblem)
if p := discovery.ForeignMemoryProblem(int(foreign[i])); p != "" {
ps = append(ps, p)
}
n.GPUs[i].Health.Problems = ps
}
}
if len(problems) > 0 {
// Problems are reported per node, not per GPU, so attach them to the
// node's first GPU rather than guessing which one is affected. Any
+13 -1
View File
@@ -255,6 +255,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
}
seen := map[string]struct{}{}
var out []web.ChatModel
// What each model's replicas compute on, across all of them: one model
// can run on a card and on CPUs at once.
hardware := map[string][]string{}
for _, s := range r.reg.Snapshots("") {
hw := hardwareOf(s.Pool)
for _, m := range s.Models {
name, _ := r.resolve(m)
if hw != "" && !slices.Contains(hardware[name], hw) {
hardware[name] = append(hardware[name], hw)
}
}
}
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
// Resolved rather than taken as sent: an alias and the name it
@@ -265,7 +277,7 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
}
seen[name] = struct{}{}
if r.permits(principal, name) {
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()})
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated(), Hardware: hardware[name]})
}
}
}
+50
View File
@@ -0,0 +1,50 @@
package router
import (
"net/http"
"net/http/httptest"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func TestAReplySaysWhatHardwareAnsweredIt(t *testing.T) {
// qwen2.5-coder-14b had one replica on cuda01's card and one on a CPU
// node, and the chat page could not tell which had answered: the same
// model, ten times faster or slower, for no visible reason.
for pool, want := range map[string]string{
"none/6000-24g": "6000-24g",
"none/cpu": "cpu",
"": "",
} {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig())
rt.SetEndpoints(t.Context(), []endpoint.Info{
{ID: "r", URL: e.srv.URL, Models: []string{"m"}, Pool: pool, Weight: 1},
})
waitForEndpoint(t, rt)
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%q: status %d: %s", pool, rec.Code, rec.Body.String())
}
if got := rec.Header().Get(hardwareHeader); got != want {
t.Errorf("pool %q: hardware %q, want %q", pool, got, want)
}
}
}
func TestThePickerSaysWhereEachModelRuns(t *testing.T) {
gpu, cpu := newFakeEngine(t, "qwen2.5-coder-7b-instruct"), newFakeEngine(t, "qwen2.5-coder-7b-instruct")
rt, user := chatRouter(t, gpu.srv.URL)
rt.SetEndpoints(t.Context(), []endpoint.Info{
{ID: "g", URL: gpu.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, Pool: "none/6000-24g", Weight: 1},
{ID: "c", URL: cpu.srv.URL, Models: []string{"coder"}, Pool: "none/cpu", Weight: 1},
})
waitForEndpoint(t, rt)
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
if len(got) != 1 || got[0].Runs() != "GPU + CPU" {
t.Fatalf("picker %+v; want one model running on GPU + CPU, its alias's replica included", got)
}
}
+25
View File
@@ -749,6 +749,16 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
if a, ok := r.adapter(req.Model); ok && a.base == model {
body = withAdapter(body, model, a.index, a.scale)
}
// What computes the answer, said before the proxy writes the headers.
// A model can have replicas on cards and on CPUs at once, and the two
// differ tenfold in speed: a reply that came slowly is otherwise
// indistinguishable from a model that is slow everywhere. Set again on
// a retry, which only happens before anything was written.
if hw := hardwareOf(dec.Endpoint.Pool); hw != "" {
w.Header().Set(hardwareHeader, hw)
} else {
w.Header().Del(hardwareHeader)
}
out, perr := r.px.Do(uctx, w, proxy.Request{
URL: dec.Endpoint.URL + req.Path,
APIKey: dec.Endpoint.APIKey,
@@ -791,6 +801,21 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
return last
}
// hardwareHeader says what the replica that answered computes on: "cpu", or
// the GPU class, such as "6000-24g".
const hardwareHeader = "x-llmgw-hardware"
// hardwareOf reads it from a pool, which placement names fabric/class —
// "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint
// configured by hand, which has no pool and says nothing about its hardware.
func hardwareOf(pool string) string {
i := strings.LastIndexByte(pool, '/')
if i < 0 || i == len(pool)-1 {
return ""
}
return pool[i+1:]
}
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
}
+27
View File
@@ -18,6 +18,33 @@ type ChatModel struct {
// offered anyway: the first message is what asks, so hiding it until it
// runs meant it could never be started from here.
OnDemand bool
// Hardware is what the model's running replicas compute on: "cpu", or a
// GPU class. Several when it runs on both, and empty when nothing runs
// or nothing said.
Hardware []string
}
// Runs names the hardware for the picker: "GPU", "CPU", or "GPU + CPU" for a
// model with replicas on both, where which one answers is the router's choice
// and each reply says which it was.
func (m ChatModel) Runs() string {
var gpu, cpu bool
for _, h := range m.Hardware {
if h == "cpu" {
cpu = true
} else {
gpu = true
}
}
switch {
case gpu && cpu:
return "GPU + CPU"
case gpu:
return "GPU"
case cpu:
return "CPU"
}
return ""
}
// ChatContext is one thing the chat page can spend against: a project, or the
+16
View File
@@ -185,3 +185,19 @@ func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) {
t.Errorf("the reply is written as HTML other than from /chat/render (%d assignments)", n)
}
}
func TestChatModelRunsNamesTheHardware(t *testing.T) {
for _, c := range []struct {
hw []string
want string
}{
{nil, ""},
{[]string{"cpu"}, "CPU"},
{[]string{"6000-24g"}, "GPU"},
{[]string{"cpu", "6000-24g"}, "GPU + CPU"},
} {
if got := (ChatModel{Hardware: c.hw}).Runs(); got != c.want {
t.Errorf("%v: %q, want %q", c.hw, got, c.want)
}
}
}
+6 -3
View File
@@ -29,7 +29,7 @@
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
@@ -301,10 +301,13 @@ access to it.</p>
let busy = null;
// A line under a reply: how long it took and how fast it came.
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
const p = document.createElement("p");
p.className = "note chat-stats";
const parts = [];
// What answered: a model with replicas on a card and on CPUs is served
// by either, and the speed below means nothing without knowing which.
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
if (tokens) parts.push(tokens + " tokens");
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
@@ -428,7 +431,7 @@ access to it.</p>
note.textContent = "Cut off at the length limit — ask it to continue.";
out.after(note);
}
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
// Once, at the end. Re-rendering every chunk would try to typeset half
// an expression on each token and flicker its way through the reply.
await renderReply(out, reply);