Files
llmgw/internal/control/agentsvc/heartbeat_test.go
T
Ilker ManapandClaude Opus 5.5 daed835d88 Measure each node's CPU and memory, and give every node a page
The nodes are CPU machines, and nothing measured how busy one was: the
heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the
total. The agent now reads /proc between heartbeats - CPU across all
cores, load averages, memory used and available, and each engine's
process group's cores and resident memory - and sends it with the
heartbeat. Nothing new runs on the nodes and Prometheus needs no new
target: the control plane exports it with the rest of the fleet, as
llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load,
llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes,
llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more
than a minute old is left out rather than repeated.

The cluster page's node table shows CPU and RAM as bars, and each node
links to a page of its own: current CPU, RAM, load and disk, the engines
on it with their cores and memory, and an hour of CPU, RAM, load,
per-model cores and memory and queue depth from Prometheus. It refreshes
in place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 23:21:23 +02:00

213 lines
7.2 KiB
Go

package agentsvc
import (
"log/slog"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) }
func newTestService() *Service {
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{},
log: slog.New(slog.DiscardHandler)}
}
func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) {
// It was sampled once, at registration, and never again. Found on the live
// cluster: an agent running for two days still reported the disk it had
// when it started, and a file deleted in between was still being offered
// for deletion. Disk is a fact about a moment, and the node whose figure
// matters most is the one busy filling up.
s := newTestService()
nodeWith(s, inventory.Node{
ID: "n1",
State: inventory.Ready,
Storage: []discovery.Storage{
{Path: "/models", FreeBytes: 200 << 30, TotalBytes: 400 << 30},
},
})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1",
Storage: []*llmgwv1.Storage{
{Path: "/models", FreeBytes: 5 << 30, TotalBytes: 400 << 30},
},
})
n, _ := s.inv.Get("n1")
free, total := n.LocalDisk()
if free != 5<<30 || total != 400<<30 {
t.Errorf("disk is %d of %d, want the figure the heartbeat carried", free, total)
}
}
func TestAnEmptyWeightListOnlyCountsWhenTheAgentMeansIt(t *testing.T) {
// Most heartbeats carry no file list because nothing changed. Treating
// that as "this node holds nothing" would empty a node's inventory five
// seconds after it registered, and placement would stop preferring the
// node that actually has the model.
s := newTestService()
nodeWith(s, inventory.Node{
ID: "n1",
State: inventory.Ready,
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
})
// Silence about weights: the list must survive.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1",
Storage: []*llmgwv1.Storage{{Path: "/models", FreeBytes: 1, TotalBytes: 2}},
})
if n, _ := s.inv.Get("n1"); len(n.Weights) != 1 {
t.Fatalf("a heartbeat with no weight list emptied the inventory: %+v", n.Weights)
}
// And when the agent says it means an empty list, it is taken.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", WeightsChanged: true})
if n, _ := s.inv.Get("n1"); len(n.Weights) != 0 {
t.Errorf("a deliberate empty list was ignored: %+v", n.Weights)
}
}
func TestANewlyFetchedModelAppearsWithoutARestart(t *testing.T) {
// The other direction of the same bug: a model fetched after registration
// was invisible until the agent restarted, so placement could not prefer
// the node that had just spent ten minutes downloading it.
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1",
WeightsChanged: true,
Weights: []*llmgwv1.LocalWeights{
{Path: "/models/new.gguf", SizeBytes: 4 << 30},
},
})
n, _ := s.inv.Get("n1")
if !n.HasWeights("/models/new.gguf") {
t.Errorf("a model fetched after registration did not appear: %+v", n.Weights)
}
}
func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) {
s := newTestService()
before := inventory.Node{
ID: "n1",
State: inventory.Ready,
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
Storage: []discovery.Storage{{Path: "/models", FreeBytes: 7, TotalBytes: 9}},
}
nodeWith(s, before)
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
n, _ := s.inv.Get("n1")
if len(n.Weights) != 1 {
t.Errorf("weights changed: %+v", n.Weights)
}
if free, _ := n.LocalDisk(); free != 7 {
t.Errorf("disk changed: %d", free)
}
}
func TestAHeartbeatCarriesMemoryAddedToARunningNode(t *testing.T) {
// Read once at registration, a node that was given 32 GB went on looking
// like the 7 GB machine it had been, and every model that needed the
// difference stayed unplaced until somebody restarted its agent.
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 32 * 1024})
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
t.Errorf("memory is %d MB, expected the 32 GB the node now has", n.MemoryTotalMB)
}
// Zero is an agent that could not read it, not a node that lost its RAM.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
t.Errorf("a heartbeat without memory erased it: %d", n.MemoryTotalMB)
}
}
func TestAHeartbeatCarriesAnEngineInstalledAfterRegistration(t *testing.T) {
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready,
EngineVersions: map[string]string{"llamacpp": "unknown"}})
// Unchanged engines are not resent, and silence must not clear them.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 1024})
if n, _ := s.inv.Get("n1"); len(n.EngineVersions) != 1 {
t.Fatalf("engines changed without being reported: %v", n.EngineVersions)
}
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1", EnginesChanged: true,
EngineVersions: map[string]string{"llamacpp": "unknown", "vllm": "unknown"},
})
n, _ := s.inv.Get("n1")
if _, ok := n.EngineVersions["vllm"]; !ok {
t.Errorf("the newly installed engine is not known: %v", n.EngineVersions)
}
}
func TestALogRequestIsAnsweredByTheNodeItWasSentTo(t *testing.T) {
s := newTestService()
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
s.conns["n1"] = c
type result struct {
lines, sources []string
err error
}
done := make(chan result, 1)
go func() {
l, src, err := s.Logs(t.Context(), "n1", "agent", 10)
done <- result{l, src, err}
}()
req := (<-c.send).GetLogs()
if req.GetSource() != "agent" || req.GetLines() != 10 {
t.Fatalf("sent %v", req)
}
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Logs{Logs: &llmgwv1.LogsReply{
RequestId: req.GetRequestId(), Lines: []string{"hello"}, Sources: []string{"agent", "shard-1"},
}}})
r := <-done
if r.err != nil || len(r.lines) != 1 || r.lines[0] != "hello" || len(r.sources) != 2 {
t.Errorf("got %+v", r)
}
if _, _, err := s.Logs(t.Context(), "nobody", "agent", 10); err == nil {
t.Error("a node with no stream answered")
}
}
func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) {
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", Load: &llmgwv1.NodeLoad{
CpuPercent: 62.5, CpuCount: 8, Load1: 3.2, MemoryUsedBytes: 6 << 30, MemoryAvailableBytes: 2 << 30,
Shards: []*llmgwv1.ShardLoad{{ShardId: "r1-0", CpuCores: 4.5, MemoryRssBytes: 5 << 30}},
}})
n, _ := s.inv.Get("n1")
if n.Load == nil || n.Load.CPUPercent != 62.5 || n.Load.MemUsedBytes != 6<<30 ||
n.Load.Shards["r1-0"].CPUCores != 4.5 {
t.Fatalf("load = %+v", n.Load)
}
// A heartbeat that measured nothing — the agent's first — leaves the last
// measurement standing rather than blanking it.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
if n, _ := s.inv.Get("n1"); n.Load == nil {
t.Error("a heartbeat without load erased it")
}
}