The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
213 lines
7.2 KiB
Go
213 lines
7.2 KiB
Go
package agentsvc
|
|
|
|
import (
|
|
"log/slog"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) }
|
|
|
|
func newTestService() *Service {
|
|
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{},
|
|
log: slog.New(slog.DiscardHandler)}
|
|
}
|
|
|
|
func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) {
|
|
// It was sampled once, at registration, and never again. Found on the live
|
|
// cluster: an agent running for two days still reported the disk it had
|
|
// when it started, and a file deleted in between was still being offered
|
|
// for deletion. Disk is a fact about a moment, and the node whose figure
|
|
// matters most is the one busy filling up.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Storage: []discovery.Storage{
|
|
{Path: "/models", FreeBytes: 200 << 30, TotalBytes: 400 << 30},
|
|
},
|
|
})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
Storage: []*llmgwv1.Storage{
|
|
{Path: "/models", FreeBytes: 5 << 30, TotalBytes: 400 << 30},
|
|
},
|
|
})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
free, total := n.LocalDisk()
|
|
if free != 5<<30 || total != 400<<30 {
|
|
t.Errorf("disk is %d of %d, want the figure the heartbeat carried", free, total)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyWeightListOnlyCountsWhenTheAgentMeansIt(t *testing.T) {
|
|
// Most heartbeats carry no file list because nothing changed. Treating
|
|
// that as "this node holds nothing" would empty a node's inventory five
|
|
// seconds after it registered, and placement would stop preferring the
|
|
// node that actually has the model.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
|
|
})
|
|
|
|
// Silence about weights: the list must survive.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
Storage: []*llmgwv1.Storage{{Path: "/models", FreeBytes: 1, TotalBytes: 2}},
|
|
})
|
|
if n, _ := s.inv.Get("n1"); len(n.Weights) != 1 {
|
|
t.Fatalf("a heartbeat with no weight list emptied the inventory: %+v", n.Weights)
|
|
}
|
|
|
|
// And when the agent says it means an empty list, it is taken.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", WeightsChanged: true})
|
|
if n, _ := s.inv.Get("n1"); len(n.Weights) != 0 {
|
|
t.Errorf("a deliberate empty list was ignored: %+v", n.Weights)
|
|
}
|
|
}
|
|
|
|
func TestANewlyFetchedModelAppearsWithoutARestart(t *testing.T) {
|
|
// The other direction of the same bug: a model fetched after registration
|
|
// was invisible until the agent restarted, so placement could not prefer
|
|
// the node that had just spent ten minutes downloading it.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
WeightsChanged: true,
|
|
Weights: []*llmgwv1.LocalWeights{
|
|
{Path: "/models/new.gguf", SizeBytes: 4 << 30},
|
|
},
|
|
})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
if !n.HasWeights("/models/new.gguf") {
|
|
t.Errorf("a model fetched after registration did not appear: %+v", n.Weights)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) {
|
|
s := newTestService()
|
|
before := inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
|
|
Storage: []discovery.Storage{{Path: "/models", FreeBytes: 7, TotalBytes: 9}},
|
|
}
|
|
nodeWith(s, before)
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
if len(n.Weights) != 1 {
|
|
t.Errorf("weights changed: %+v", n.Weights)
|
|
}
|
|
if free, _ := n.LocalDisk(); free != 7 {
|
|
t.Errorf("disk changed: %d", free)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatCarriesMemoryAddedToARunningNode(t *testing.T) {
|
|
// Read once at registration, a node that was given 32 GB went on looking
|
|
// like the 7 GB machine it had been, and every model that needed the
|
|
// difference stayed unplaced until somebody restarted its agent.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 32 * 1024})
|
|
|
|
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
|
|
t.Errorf("memory is %d MB, expected the 32 GB the node now has", n.MemoryTotalMB)
|
|
}
|
|
|
|
// Zero is an agent that could not read it, not a node that lost its RAM.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
|
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
|
|
t.Errorf("a heartbeat without memory erased it: %d", n.MemoryTotalMB)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatCarriesAnEngineInstalledAfterRegistration(t *testing.T) {
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready,
|
|
EngineVersions: map[string]string{"llamacpp": "unknown"}})
|
|
|
|
// Unchanged engines are not resent, and silence must not clear them.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 1024})
|
|
if n, _ := s.inv.Get("n1"); len(n.EngineVersions) != 1 {
|
|
t.Fatalf("engines changed without being reported: %v", n.EngineVersions)
|
|
}
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1", EnginesChanged: true,
|
|
EngineVersions: map[string]string{"llamacpp": "unknown", "vllm": "unknown"},
|
|
})
|
|
n, _ := s.inv.Get("n1")
|
|
if _, ok := n.EngineVersions["vllm"]; !ok {
|
|
t.Errorf("the newly installed engine is not known: %v", n.EngineVersions)
|
|
}
|
|
}
|
|
|
|
func TestALogRequestIsAnsweredByTheNodeItWasSentTo(t *testing.T) {
|
|
s := newTestService()
|
|
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
|
|
s.conns["n1"] = c
|
|
|
|
type result struct {
|
|
lines, sources []string
|
|
err error
|
|
}
|
|
done := make(chan result, 1)
|
|
go func() {
|
|
l, src, err := s.Logs(t.Context(), "n1", "agent", 10)
|
|
done <- result{l, src, err}
|
|
}()
|
|
|
|
req := (<-c.send).GetLogs()
|
|
if req.GetSource() != "agent" || req.GetLines() != 10 {
|
|
t.Fatalf("sent %v", req)
|
|
}
|
|
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Logs{Logs: &llmgwv1.LogsReply{
|
|
RequestId: req.GetRequestId(), Lines: []string{"hello"}, Sources: []string{"agent", "shard-1"},
|
|
}}})
|
|
|
|
r := <-done
|
|
if r.err != nil || len(r.lines) != 1 || r.lines[0] != "hello" || len(r.sources) != 2 {
|
|
t.Errorf("got %+v", r)
|
|
}
|
|
|
|
if _, _, err := s.Logs(t.Context(), "nobody", "agent", 10); err == nil {
|
|
t.Error("a node with no stream answered")
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) {
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", Load: &llmgwv1.NodeLoad{
|
|
CpuPercent: 62.5, CpuCount: 8, Load1: 3.2, MemoryUsedBytes: 6 << 30, MemoryAvailableBytes: 2 << 30,
|
|
Shards: []*llmgwv1.ShardLoad{{ShardId: "r1-0", CpuCores: 4.5, MemoryRssBytes: 5 << 30}},
|
|
}})
|
|
n, _ := s.inv.Get("n1")
|
|
if n.Load == nil || n.Load.CPUPercent != 62.5 || n.Load.MemUsedBytes != 6<<30 ||
|
|
n.Load.Shards["r1-0"].CPUCores != 4.5 {
|
|
t.Fatalf("load = %+v", n.Load)
|
|
}
|
|
|
|
// A heartbeat that measured nothing — the agent's first — leaves the last
|
|
// measurement standing rather than blanking it.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
|
if n, _ := s.inv.Get("n1"); n.Load == nil {
|
|
t.Error("a heartbeat without load erased it")
|
|
}
|
|
}
|