Ilker ManapandClaude Opus 5.5
daed835d88
The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
97 lines
2.9 KiB
Go
97 lines
2.9 KiB
Go
package discovery
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fakeProc is a /proc with two CPUs, a memory picture and processes.
|
|
type fakeProc struct {
|
|
t *testing.T
|
|
root string
|
|
}
|
|
|
|
func newFakeProc(t *testing.T) *fakeProc {
|
|
f := &fakeProc{t: t, root: t.TempDir()}
|
|
f.write("loadavg", "1.50 0.75 0.25 2/300 4242\n")
|
|
f.write("meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 3000000 kB\n")
|
|
return f
|
|
}
|
|
|
|
func (f *fakeProc) write(name, body string) {
|
|
f.t.Helper()
|
|
p := filepath.Join(f.root, name)
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
|
f.t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// cpu sets the aggregate counters: user and idle, the rest zero.
|
|
func (f *fakeProc) cpu(user, idle uint64) {
|
|
f.write("stat", fmt.Sprintf("cpu %d 0 0 %d 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n", user, idle))
|
|
}
|
|
|
|
// proc sets one process: its group, CPU jiffies (as utime) and RSS in pages.
|
|
func (f *fakeProc) proc(pid, pgrp int, jiffies uint64, rss int64) {
|
|
// The name holds a space and a parenthesis, as real ones can.
|
|
f.write(fmt.Sprintf("%d/stat", pid), fmt.Sprintf(
|
|
"%d (llama-server (x)) S 1 %d %d 0 -1 0 0 0 0 0 %d 0 0 0 20 0 1 0 100 1000 %d 0\n",
|
|
pid, pgrp, pgrp, jiffies, rss))
|
|
}
|
|
|
|
func TestLoadIsTheDifferenceBetweenTwoReadings(t *testing.T) {
|
|
p := newFakeProc(t)
|
|
at := time.Unix(1000, 0)
|
|
s := newLoadSampler(p.root, func() time.Time { return at })
|
|
|
|
p.cpu(100, 900)
|
|
p.proc(42, 42, 1000, 10)
|
|
p.proc(43, 42, 500, 5) // a worker in the engine's group
|
|
p.proc(7, 7, 99999, 1) // something else entirely
|
|
if _, ok := s.Sample(map[string]int{"shard-a": 42}); ok {
|
|
t.Fatal("the first reading has nothing to compare with and should report nothing")
|
|
}
|
|
|
|
// Two seconds later: 300 of the 400 jiffies that passed were busy, and
|
|
// the engine's group used 400 of them.
|
|
at = at.Add(2 * time.Second)
|
|
p.cpu(400, 1000)
|
|
p.proc(42, 42, 1300, 10)
|
|
p.proc(43, 42, 600, 6)
|
|
l, ok := s.Sample(map[string]int{"shard-a": 42})
|
|
if !ok {
|
|
t.Fatal("the second reading should report")
|
|
}
|
|
if l.CPUPercent != 75 || l.CPUCount != 2 {
|
|
t.Errorf("cpu %.1f%% of %d, want 75%% of 2", l.CPUPercent, l.CPUCount)
|
|
}
|
|
if l.Load1 != 1.5 || l.Load15 != 0.25 {
|
|
t.Errorf("load %v %v", l.Load1, l.Load15)
|
|
}
|
|
if l.MemUsed != 5000000*1024 || l.MemAvail != 3000000*1024 {
|
|
t.Errorf("memory used %d available %d", l.MemUsed, l.MemAvail)
|
|
}
|
|
if len(l.Shards) != 1 {
|
|
t.Fatalf("shards = %+v", l.Shards)
|
|
}
|
|
sh := l.Shards[0]
|
|
// 400 jiffies over 2 seconds at 100 a second is two cores.
|
|
if sh.ID != "shard-a" || sh.CPUCores != 2 || sh.RSSBytes != 16*int64(os.Getpagesize()) {
|
|
t.Errorf("shard = %+v", sh)
|
|
}
|
|
}
|
|
|
|
func TestNoProcMeansNoLoad(t *testing.T) {
|
|
s := newLoadSampler(filepath.Join(t.TempDir(), "missing"), time.Now)
|
|
s.Sample(nil)
|
|
if _, ok := s.Sample(nil); ok {
|
|
t.Error("a system without /proc should report nothing, not zeros")
|
|
}
|
|
}
|