Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b89d5cdea | ||
|
|
477d0bf19d |
No files matched your search
@@ -75,6 +75,33 @@ and a figure the driver does not give is left out rather than shown as zero.
|
||||
each card and whose quota it counts against, for joining the GPU series to
|
||||
models and tenants.
|
||||
|
||||
### The chat page says whether a GPU or a CPU answered
|
||||
|
||||
A model can have replicas on a card and on CPUs at once, and the two answer
|
||||
at very different speeds; the chat page gave no way to tell which one a reply
|
||||
came from. The picker now says where each model runs — `GPU`, `CPU` or
|
||||
`GPU + CPU` — and the line under each reply names what answered it: `GPU
|
||||
6000-24g` or `CPU`, beside its tokens a second.
|
||||
|
||||
Every answer through the gateway carries the same as `x-llmgw-hardware`:
|
||||
`cpu`, or the GPU class. An endpoint configured by hand, with no pool, sends
|
||||
none.
|
||||
|
||||
### A card shared with something else is judged every heartbeat
|
||||
|
||||
A card with another process on it when its agent started was kept out of
|
||||
placement, and stayed out — "5014 MiB already allocated by another process" —
|
||||
after that process let go of it, until the agent was restarted. Found on a
|
||||
node shared with ComfyUI. The other way round, a tenant arriving after
|
||||
registration was never seen, and a card the agent's own engine held when the
|
||||
agent restarted read as someone else's.
|
||||
|
||||
The agent now reads every card's memory per process on each heartbeat,
|
||||
leaves out what its own engines hold, and sends the rest; the control plane
|
||||
keeps a card out of placement on that reading, and lets it back in when it
|
||||
falls under 512 MiB. `llmgw_gpu_memory_used_bytes` reads the same figure. An
|
||||
agent too old to send it keeps the reading from registration.
|
||||
|
||||
### A GPU placement no longer runs on the CPU unnoticed
|
||||
|
||||
llama.cpp placed on a GPU, with its CUDA backend unable to load (cuBLAS not
|
||||
|
||||
@@ -2336,9 +2336,17 @@ type Heartbeat struct {
|
||||
GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
|
||||
Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
|
||||
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
||||
CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
|
||||
// gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
|
||||
// this agent's shards hold on each card, read afresh every heartbeat. What
|
||||
// keeps a card out of placement for another tenant is decided from this
|
||||
// rather than from registration: a card that had something else on it when
|
||||
// the agent started is not taken forever, and one that gains a tenant later
|
||||
// is not handed out. Empty from an agent that does not measure it, or from
|
||||
// one that could not this time.
|
||||
GpuForeignMemoryMb []int32 `protobuf:"varint,19,rep,packed,name=gpu_foreign_memory_mb,json=gpuForeignMemoryMb,proto3" json:"gpu_foreign_memory_mb,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Heartbeat) Reset() {
|
||||
@@ -2497,6 +2505,13 @@ func (x *Heartbeat) GetCpuCount() int32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetGpuForeignMemoryMb() []int32 {
|
||||
if x != nil {
|
||||
return x.GpuForeignMemoryMb
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
// versions. Nothing more.
|
||||
//
|
||||
@@ -4260,7 +4275,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x06status\x18\x03 \x01(\tR\x06status\x12!\n" +
|
||||
"\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" +
|
||||
"\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" +
|
||||
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xf6\x06\n" +
|
||||
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xa9\a\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
@@ -4281,7 +4296,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" +
|
||||
"\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" +
|
||||
"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x1aA\n" +
|
||||
"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x121\n" +
|
||||
"\x15gpu_foreign_memory_mb\x18\x13 \x03(\x05R\x12gpuForeignMemoryMb\x1aA\n" +
|
||||
"\x13EngineVersionsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
|
||||
|
||||
@@ -474,6 +474,15 @@ message Heartbeat {
|
||||
IMEX imex = 17;
|
||||
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
||||
int32 cpu_count = 18;
|
||||
|
||||
// gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
|
||||
// this agent's shards hold on each card, read afresh every heartbeat. What
|
||||
// keeps a card out of placement for another tenant is decided from this
|
||||
// rather than from registration: a card that had something else on it when
|
||||
// the agent started is not taken forever, and one that gains a tenant later
|
||||
// is not handed out. Empty from an agent that does not measure it, or from
|
||||
// one that could not this time.
|
||||
repeated int32 gpu_foreign_memory_mb = 19;
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
|
||||
+18
-2
@@ -591,8 +591,24 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
for _, g := range inv.GPUs {
|
||||
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
|
||||
// Measured now, not carried from registration: a card is shared with
|
||||
// whatever else runs on the machine, and that comes and goes while the
|
||||
// agent runs. The registration figure stays only for a node where
|
||||
// nvidia-smi cannot be asked.
|
||||
if len(inv.GPUs) > 0 {
|
||||
var shards []int
|
||||
for _, st := range a.sup.Statuses() {
|
||||
if st.PID > 0 {
|
||||
shards = append(shards, st.PID)
|
||||
}
|
||||
}
|
||||
hb.GpuForeignMemoryMb, hb.GpuMemoryUsedMb = a.foreignGPUMemory(context.Background(), shards)
|
||||
}
|
||||
if len(hb.GpuMemoryUsedMb) != len(inv.GPUs) {
|
||||
hb.GpuForeignMemoryMb, hb.GpuMemoryUsedMb = nil, nil
|
||||
for _, g := range inv.GPUs {
|
||||
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
|
||||
}
|
||||
}
|
||||
|
||||
// Rescanned here rather than reused from registration. Disk is not a fact
|
||||
|
||||
@@ -106,9 +106,27 @@ func parseSMI(out string) ([]GPU, error) {
|
||||
return gpus, nil
|
||||
}
|
||||
|
||||
// residentMemoryToleranceMB is how much allocated memory is written off as
|
||||
// ForeignMemoryToleranceMB is how much allocated memory is written off as
|
||||
// driver overhead rather than a foreign process.
|
||||
const residentMemoryToleranceMB = 512
|
||||
const ForeignMemoryToleranceMB = 512
|
||||
|
||||
// foreignMemorySuffix ends the problem a card with another tenant on it is
|
||||
// kept out of placement for. One phrase, so that the control plane can find
|
||||
// it again and drop it when the tenant leaves.
|
||||
const foreignMemorySuffix = " MiB already allocated by another process"
|
||||
|
||||
// ForeignMemoryProblem is the problem for mb of memory held by something
|
||||
// that is not one of this agent's engines, or "" when that is too little to
|
||||
// count.
|
||||
func ForeignMemoryProblem(mb int) string {
|
||||
if mb <= ForeignMemoryToleranceMB {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%d%s", mb, foreignMemorySuffix)
|
||||
}
|
||||
|
||||
// IsForeignMemoryProblem reports whether a problem is ForeignMemoryProblem's.
|
||||
func IsForeignMemoryProblem(p string) bool { return strings.HasSuffix(p, foreignMemorySuffix) }
|
||||
|
||||
// assessHealth flags conditions that should keep a GPU out of placement.
|
||||
func assessHealth(g *GPU) {
|
||||
@@ -119,8 +137,12 @@ func assessHealth(g *GPU) {
|
||||
// Memory already in use means something else is on the card. Starting an
|
||||
// engine there would either fail to allocate or quietly evict the other
|
||||
// process's throughput.
|
||||
if g.MemoryUsedMB > residentMemoryToleranceMB {
|
||||
g.Health.Problems = append(g.Health.Problems,
|
||||
fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
|
||||
//
|
||||
// Only a first reading: this runs before the agent knows which processes
|
||||
// are its own, so an engine still running from before a restart reads as
|
||||
// foreign here. Every heartbeat measures it again, without the agent's
|
||||
// own engines, and the control plane replaces this with that.
|
||||
if p := ForeignMemoryProblem(g.MemoryUsedMB); p != "" {
|
||||
g.Health.Problems = append(g.Health.Problems, p)
|
||||
}
|
||||
}
|
||||
@@ -34,13 +34,47 @@ func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
|
||||
return parseGPUProcesses(out)
|
||||
}
|
||||
|
||||
// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
|
||||
// is allocated on it, and which processes hold how much of that.
|
||||
type GPUMemory struct {
|
||||
UsedMB int
|
||||
Procs []GPUProcess
|
||||
}
|
||||
|
||||
// GPUProcess is one process holding memory on a card.
|
||||
type GPUProcess struct {
|
||||
PID int
|
||||
UsedMB int
|
||||
}
|
||||
|
||||
// GPUMemoryByProcess reports each GPU's memory, by index, with the processes
|
||||
// holding it — from the same report as GPUProcesses, so the two never
|
||||
// disagree about who is on a card.
|
||||
//
|
||||
// A node with no nvidia-smi returns nil and no error, as discovery does.
|
||||
func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) {
|
||||
if binary == "" {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
binary = "nvidia-smi"
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("running %s: %w", binary, err)
|
||||
}
|
||||
return parseGPUMemory(out)
|
||||
}
|
||||
|
||||
// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
|
||||
// index order, and the processes on it.
|
||||
// index order, its memory and the processes on it.
|
||||
type smiLog struct {
|
||||
GPUs []struct {
|
||||
Used string `xml:"fb_memory_usage>used"`
|
||||
Processes []struct {
|
||||
PID string `xml:"pid"`
|
||||
Type string `xml:"type"`
|
||||
PID string `xml:"pid"`
|
||||
Type string `xml:"type"`
|
||||
UsedMB string `xml:"used_memory"`
|
||||
} `xml:"processes>process_info"`
|
||||
} `xml:"gpu"`
|
||||
}
|
||||
@@ -48,17 +82,39 @@ type smiLog struct {
|
||||
// parseGPUProcesses reads the report. GPUs are listed in index order, which
|
||||
// is nvidia-smi's own numbering — the one placement assigns by.
|
||||
func parseGPUProcesses(report []byte) (map[int][]int, error) {
|
||||
mem, err := parseGPUMemory(report)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[int][]int{}
|
||||
for i, g := range mem {
|
||||
for _, p := range g.Procs {
|
||||
out[i] = append(out[i], p.PID)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// parseGPUMemory reads the report into each GPU's memory and its holders.
|
||||
func parseGPUMemory(report []byte) ([]GPUMemory, error) {
|
||||
var log smiLog
|
||||
if err := xml.Unmarshal(report, &log); err != nil {
|
||||
return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
|
||||
}
|
||||
out := map[int][]int{}
|
||||
out := make([]GPUMemory, len(log.GPUs))
|
||||
for i, g := range log.GPUs {
|
||||
out[i].UsedMB = mib(g.Used)
|
||||
for _, p := range g.Processes {
|
||||
if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
|
||||
out[i] = append(out[i], pid)
|
||||
out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mib reads "4932 MiB"; anything else, "N/A" included, is zero.
|
||||
func mib(s string) int {
|
||||
n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB")))
|
||||
return n
|
||||
}
|
||||
@@ -36,3 +36,43 @@ func TestGPUProcessesOfEveryKindAreListedByIndex(t *testing.T) {
|
||||
t.Error("a report that does not parse was read as no processes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMemoryIsReadPerProcess(t *testing.T) {
|
||||
report := `<?xml version="1.0" ?>
|
||||
<nvidia_smi_log>
|
||||
<gpu id="00000000:01:00.0"><product_name>Quadro RTX 6000</product_name>
|
||||
<fb_memory_usage><total>24576 MiB</total><used>14514 MiB</used><free>10062 MiB</free></fb_memory_usage>
|
||||
<processes>
|
||||
<process_info><pid>71315</pid><type>C</type><used_memory>4932 MiB</used_memory></process_info>
|
||||
<process_info><pid>4100</pid><type>C</type><used_memory>9500 MiB</used_memory></process_info>
|
||||
</processes>
|
||||
</gpu>
|
||||
<gpu id="00000000:02:00.0">
|
||||
<fb_memory_usage><used>N/A</used></fb_memory_usage>
|
||||
<processes></processes>
|
||||
</gpu>
|
||||
</nvidia_smi_log>`
|
||||
got, err := parseGPUMemory([]byte(report))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []GPUMemory{
|
||||
{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
|
||||
{},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForeignMemoryBelowTheToleranceIsNoProblem(t *testing.T) {
|
||||
// A CUDA context that has let go of its model still holds a few hundred
|
||||
// MiB: ComfyUI after /free held 324.
|
||||
if p := ForeignMemoryProblem(330); p != "" {
|
||||
t.Errorf("330 MiB is a problem: %q", p)
|
||||
}
|
||||
p := ForeignMemoryProblem(5014)
|
||||
if !IsForeignMemoryProblem(p) || IsForeignMemoryProblem("2 uncorrected ECC errors") {
|
||||
t.Errorf("%q is not recognised as the foreign memory problem, or ECC is", p)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
// nvidia-smi and /proc.
|
||||
var (
|
||||
gpuProcesses = discovery.GPUProcesses
|
||||
gpuMemory = discovery.GPUMemoryByProcess
|
||||
processGroup = pgidOf
|
||||
// gpuCheckWait is how long a shard has, once it answers, to show up on
|
||||
// its GPUs: the weights are on them by then, but nvidia-smi can lag.
|
||||
@@ -75,6 +76,44 @@ func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.
|
||||
"or run `llmgw doctor` on this node", joinIndices(missing), seen)
|
||||
}
|
||||
|
||||
// foreignGPUMemory is what each GPU holds for anything other than this
|
||||
// agent's shards, by index, and each GPU's memory in use. Nil when nvidia-smi
|
||||
// cannot be asked, which is no reading rather than an empty card.
|
||||
//
|
||||
// Everything allocated less what the shards' processes hold, rather than the
|
||||
// sum of the other processes: a process in another container's PID namespace
|
||||
// holds memory nvidia-smi lists under no PID this node can see, and it is
|
||||
// still not memory placement can hand out.
|
||||
func (a *Agent) foreignGPUMemory(ctx context.Context, shards []int) (foreign, used []int32) {
|
||||
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || mem == nil {
|
||||
return nil, nil
|
||||
}
|
||||
groups := make([]int, len(shards))
|
||||
for i, pid := range shards {
|
||||
groups[i] = processGroup(pid)
|
||||
}
|
||||
ours := func(p int) bool {
|
||||
for i, pid := range shards {
|
||||
if holds([]int{p}, pid, groups[i]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, g := range mem {
|
||||
f := g.UsedMB
|
||||
for _, p := range g.Procs {
|
||||
if ours(p.PID) {
|
||||
f -= p.UsedMB
|
||||
}
|
||||
}
|
||||
foreign = append(foreign, int32(max(f, 0)))
|
||||
used = append(used, int32(g.UsedMB))
|
||||
}
|
||||
return foreign, used
|
||||
}
|
||||
|
||||
// holds reports whether any of pids is the shard's process or in its group:
|
||||
// an engine that computes in a child process is still the shard.
|
||||
func holds(pids []int, pid, group int) bool {
|
||||
|
||||
@@ -3,10 +3,12 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
|
||||
@@ -89,3 +91,34 @@ func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) {
|
||||
t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
|
||||
// cuda01: ComfyUI and one of our engines on the same card. Only ComfyUI's
|
||||
// share — and whatever no listed process accounts for — is foreign.
|
||||
oldMem, oldGroup := gpuMemory, processGroup
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
|
||||
return []discovery.GPUMemory{
|
||||
{UsedMB: 15000, Procs: []discovery.GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 512, UsedMB: 9500}}},
|
||||
{UsedMB: 300},
|
||||
}, nil
|
||||
}
|
||||
processGroup = func(pid int) int {
|
||||
if pid >= 500 && pid < 600 {
|
||||
return 500
|
||||
}
|
||||
return pid
|
||||
}
|
||||
t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
|
||||
|
||||
a := &Agent{log: slog.Default()}
|
||||
foreign, used := a.foreignGPUMemory(context.Background(), []int{500})
|
||||
if !slices.Equal(foreign, []int32{5500, 300}) || !slices.Equal(used, []int32{15000, 300}) {
|
||||
t.Errorf("foreign %v, used %v; want [5500 300] and [15000 300]", foreign, used)
|
||||
}
|
||||
|
||||
// No nvidia-smi to ask is no reading, not an empty card.
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
|
||||
if foreign, _ := a.foreignGPUMemory(context.Background(), nil); foreign != nil {
|
||||
t.Errorf("no reading came back as %v", foreign)
|
||||
}
|
||||
}
|
||||
@@ -283,3 +283,36 @@ func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
|
||||
t.Errorf("the agent's refusal was not passed on: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnotherTenantOnACardIsJudgedEveryHeartbeat(t *testing.T) {
|
||||
// Found on cuda01: ComfyUI held 5 GB when the agent started, so the card
|
||||
// was kept out of placement. ComfyUI let go of it and the card stayed
|
||||
// out, saying "5014 MiB already allocated by another process", until the
|
||||
// agent was restarted.
|
||||
s := newTestService()
|
||||
nodeWith(s, inventory.Node{
|
||||
ID: "n1", State: inventory.Ready,
|
||||
GPUs: []discovery.GPU{{Index: 0, Health: discovery.GPUHealth{Problems: []string{
|
||||
discovery.ForeignMemoryProblem(5014), "2 uncorrected ECC errors"}}}},
|
||||
})
|
||||
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", GpuForeignMemoryMb: []int32{330}})
|
||||
n, _ := s.inv.Get("n1")
|
||||
if got := n.GPUs[0].Health.Problems; len(got) != 1 || got[0] != "2 uncorrected ECC errors" {
|
||||
t.Fatalf("problems are %q: the tenant left, and only the ECC errors should remain", got)
|
||||
}
|
||||
|
||||
// And the other way: a tenant arriving after registration.
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", GpuForeignMemoryMb: []int32{12000}})
|
||||
n, _ = s.inv.Get("n1")
|
||||
if got := n.GPUs[0].Health.Problems; len(got) != 2 || got[1] != discovery.ForeignMemoryProblem(12000) {
|
||||
t.Fatalf("problems are %q, want the new tenant's 12000 MiB", got)
|
||||
}
|
||||
|
||||
// An agent that sends no reading changes nothing.
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", GpuMemoryUsedMb: []int32{100}})
|
||||
n, _ = s.inv.Get("n1")
|
||||
if len(n.GPUs[0].Health.Problems) != 2 {
|
||||
t.Errorf("a heartbeat with no foreign reading cleared the card: %q", n.GPUs[0].Health.Problems)
|
||||
}
|
||||
}
|
||||
@@ -377,11 +377,12 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
// a reconnect.
|
||||
func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
used := hb.GetGpuMemoryUsedMb()
|
||||
foreign := hb.GetGpuForeignMemoryMb()
|
||||
problems := hb.GetNewProblems()
|
||||
storage := hb.GetStorage()
|
||||
memMB := int(hb.GetMemoryTotalMb())
|
||||
cpus := int(hb.GetCpuCount())
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
if len(used) == 0 && len(foreign) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
||||
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
||||
@@ -397,6 +398,20 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
n.GPUs[i].MemoryUsedMB = int(used[i])
|
||||
}
|
||||
}
|
||||
// Another tenant on a card, judged afresh: the reading at registration is
|
||||
// replaced, both ways. Without this a card that had something on it when
|
||||
// its agent started stayed out of placement after it left, and said so,
|
||||
// until the agent was restarted. Only a reading for every card replaces
|
||||
// it; an agent that sends none, or could not measure, leaves it standing.
|
||||
if len(foreign) == len(n.GPUs) {
|
||||
for i := range n.GPUs {
|
||||
ps := slices.DeleteFunc(slices.Clone(n.GPUs[i].Health.Problems), discovery.IsForeignMemoryProblem)
|
||||
if p := discovery.ForeignMemoryProblem(int(foreign[i])); p != "" {
|
||||
ps = append(ps, p)
|
||||
}
|
||||
n.GPUs[i].Health.Problems = ps
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
// Problems are reported per node, not per GPU, so attach them to the
|
||||
// node's first GPU rather than guessing which one is affected. Any
|
||||
|
||||
+13
-1
@@ -255,6 +255,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var out []web.ChatModel
|
||||
// What each model's replicas compute on, across all of them: one model
|
||||
// can run on a card and on CPUs at once.
|
||||
hardware := map[string][]string{}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
hw := hardwareOf(s.Pool)
|
||||
for _, m := range s.Models {
|
||||
name, _ := r.resolve(m)
|
||||
if hw != "" && !slices.Contains(hardware[name], hw) {
|
||||
hardware[name] = append(hardware[name], hw)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
for _, m := range s.Models {
|
||||
// Resolved rather than taken as sent: an alias and the name it
|
||||
@@ -265,7 +277,7 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
if r.permits(principal, name) {
|
||||
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated()})
|
||||
out = append(out, web.ChatModel{Name: name, Simulated: entry.Simulated(), Hardware: hardware[name]})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
func TestAReplySaysWhatHardwareAnsweredIt(t *testing.T) {
|
||||
// qwen2.5-coder-14b had one replica on cuda01's card and one on a CPU
|
||||
// node, and the chat page could not tell which had answered: the same
|
||||
// model, ten times faster or slower, for no visible reason.
|
||||
for pool, want := range map[string]string{
|
||||
"none/6000-24g": "6000-24g",
|
||||
"none/cpu": "cpu",
|
||||
"": "",
|
||||
} {
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig())
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "r", URL: e.srv.URL, Models: []string{"m"}, Pool: pool, Weight: 1},
|
||||
})
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "m", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%q: status %d: %s", pool, rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get(hardwareHeader); got != want {
|
||||
t.Errorf("pool %q: hardware %q, want %q", pool, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestThePickerSaysWhereEachModelRuns(t *testing.T) {
|
||||
gpu, cpu := newFakeEngine(t, "qwen2.5-coder-7b-instruct"), newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
||||
rt, user := chatRouter(t, gpu.srv.URL)
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "g", URL: gpu.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, Pool: "none/6000-24g", Weight: 1},
|
||||
{ID: "c", URL: cpu.srv.URL, Models: []string{"coder"}, Pool: "none/cpu", Weight: 1},
|
||||
})
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
|
||||
if len(got) != 1 || got[0].Runs() != "GPU + CPU" {
|
||||
t.Fatalf("picker %+v; want one model running on GPU + CPU, its alias's replica included", got)
|
||||
}
|
||||
}
|
||||
@@ -749,6 +749,16 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
if a, ok := r.adapter(req.Model); ok && a.base == model {
|
||||
body = withAdapter(body, model, a.index, a.scale)
|
||||
}
|
||||
// What computes the answer, said before the proxy writes the headers.
|
||||
// A model can have replicas on cards and on CPUs at once, and the two
|
||||
// differ tenfold in speed: a reply that came slowly is otherwise
|
||||
// indistinguishable from a model that is slow everywhere. Set again on
|
||||
// a retry, which only happens before anything was written.
|
||||
if hw := hardwareOf(dec.Endpoint.Pool); hw != "" {
|
||||
w.Header().Set(hardwareHeader, hw)
|
||||
} else {
|
||||
w.Header().Del(hardwareHeader)
|
||||
}
|
||||
out, perr := r.px.Do(uctx, w, proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
@@ -791,6 +801,21 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
return last
|
||||
}
|
||||
|
||||
// hardwareHeader says what the replica that answered computes on: "cpu", or
|
||||
// the GPU class, such as "6000-24g".
|
||||
const hardwareHeader = "x-llmgw-hardware"
|
||||
|
||||
// hardwareOf reads it from a pool, which placement names fabric/class —
|
||||
// "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint
|
||||
// configured by hand, which has no pool and says nothing about its hardware.
|
||||
func hardwareOf(pool string) string {
|
||||
i := strings.LastIndexByte(pool, '/')
|
||||
if i < 0 || i == len(pool)-1 {
|
||||
return ""
|
||||
}
|
||||
return pool[i+1:]
|
||||
}
|
||||
|
||||
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
||||
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
||||
}
|
||||
|
||||
@@ -18,6 +18,33 @@ type ChatModel struct {
|
||||
// offered anyway: the first message is what asks, so hiding it until it
|
||||
// runs meant it could never be started from here.
|
||||
OnDemand bool
|
||||
// Hardware is what the model's running replicas compute on: "cpu", or a
|
||||
// GPU class. Several when it runs on both, and empty when nothing runs
|
||||
// or nothing said.
|
||||
Hardware []string
|
||||
}
|
||||
|
||||
// Runs names the hardware for the picker: "GPU", "CPU", or "GPU + CPU" for a
|
||||
// model with replicas on both, where which one answers is the router's choice
|
||||
// and each reply says which it was.
|
||||
func (m ChatModel) Runs() string {
|
||||
var gpu, cpu bool
|
||||
for _, h := range m.Hardware {
|
||||
if h == "cpu" {
|
||||
cpu = true
|
||||
} else {
|
||||
gpu = true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case gpu && cpu:
|
||||
return "GPU + CPU"
|
||||
case gpu:
|
||||
return "GPU"
|
||||
case cpu:
|
||||
return "CPU"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ChatContext is one thing the chat page can spend against: a project, or the
|
||||
|
||||
@@ -185,3 +185,19 @@ func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) {
|
||||
t.Errorf("the reply is written as HTML other than from /chat/render (%d assignments)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatModelRunsNamesTheHardware(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
hw []string
|
||||
want string
|
||||
}{
|
||||
{nil, ""},
|
||||
{[]string{"cpu"}, "CPU"},
|
||||
{[]string{"6000-24g"}, "GPU"},
|
||||
{[]string{"cpu", "6000-24g"}, "GPU + CPU"},
|
||||
} {
|
||||
if got := (ChatModel{Hardware: c.hw}).Runs(); got != c.want {
|
||||
t.Errorf("%v: %q, want %q", c.hw, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,7 @@
|
||||
{{if .Models}}
|
||||
<div class="chat-bar">
|
||||
<label class="inline-label">Model
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
</label>
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
@@ -301,10 +301,13 @@ access to it.</p>
|
||||
let busy = null;
|
||||
|
||||
// A line under a reply: how long it took and how fast it came.
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
|
||||
const p = document.createElement("p");
|
||||
p.className = "note chat-stats";
|
||||
const parts = [];
|
||||
// What answered: a model with replicas on a card and on CPUs is served
|
||||
// by either, and the speed below means nothing without knowing which.
|
||||
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
|
||||
if (tokens) parts.push(tokens + " tokens");
|
||||
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
|
||||
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
|
||||
@@ -428,7 +431,7 @@ access to it.</p>
|
||||
note.textContent = "Cut off at the length limit — ask it to continue.";
|
||||
out.after(note);
|
||||
}
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
|
||||
// Once, at the end. Re-rendering every chunk would try to typeset half
|
||||
// an expression on each token and flicker its way through the reply.
|
||||
await renderReply(out, reply);
|
||||
|
||||
Reference in new issue
Block a user