Files
llmgw/internal/control/agentsvc/heartbeat_test.go
T
Ilker ManapandClaude Opus 5.5 33e678ab84 Show a node's logs on the admin page
Seeing what a node did meant ssh through a jump host, to a machine whose
address nobody remembers, to read a journal only root may open. When an
engine install failed on a new node today, the reason was one line — and it
was four steps away from the page that said the install had failed.

Each node's row on the cluster page now links to its logs: the agent's own,
and every engine running there, chosen from the list the node itself sends.
The page can follow, refreshing every five seconds, and is on both the admin
listener and /admin. `llmgw ctl logs NODE [--source SHARD] [-n N]` reads the
same.

They travel over the stream the agent already holds, so they need nothing
the control plane does not already have: a request goes down, the lines come
back, and a reply that does not come in ten seconds is an agent too old to
answer. The agent keeps its last 2000 lines in memory as well as writing them
wherever it did before; the engines' last few hundred lines were already kept
and sent only when one crashed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 12:27:47 +02:00

191 lines
6.4 KiB
Go

package agentsvc
import (
"log/slog"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) }
func newTestService() *Service {
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{},
log: slog.New(slog.DiscardHandler)}
}
func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) {
// It was sampled once, at registration, and never again. Found on the live
// cluster: an agent running for two days still reported the disk it had
// when it started, and a file deleted in between was still being offered
// for deletion. Disk is a fact about a moment, and the node whose figure
// matters most is the one busy filling up.
s := newTestService()
nodeWith(s, inventory.Node{
ID: "n1",
State: inventory.Ready,
Storage: []discovery.Storage{
{Path: "/models", FreeBytes: 200 << 30, TotalBytes: 400 << 30},
},
})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1",
Storage: []*llmgwv1.Storage{
{Path: "/models", FreeBytes: 5 << 30, TotalBytes: 400 << 30},
},
})
n, _ := s.inv.Get("n1")
free, total := n.LocalDisk()
if free != 5<<30 || total != 400<<30 {
t.Errorf("disk is %d of %d, want the figure the heartbeat carried", free, total)
}
}
func TestAnEmptyWeightListOnlyCountsWhenTheAgentMeansIt(t *testing.T) {
// Most heartbeats carry no file list because nothing changed. Treating
// that as "this node holds nothing" would empty a node's inventory five
// seconds after it registered, and placement would stop preferring the
// node that actually has the model.
s := newTestService()
nodeWith(s, inventory.Node{
ID: "n1",
State: inventory.Ready,
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
})
// Silence about weights: the list must survive.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1",
Storage: []*llmgwv1.Storage{{Path: "/models", FreeBytes: 1, TotalBytes: 2}},
})
if n, _ := s.inv.Get("n1"); len(n.Weights) != 1 {
t.Fatalf("a heartbeat with no weight list emptied the inventory: %+v", n.Weights)
}
// And when the agent says it means an empty list, it is taken.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", WeightsChanged: true})
if n, _ := s.inv.Get("n1"); len(n.Weights) != 0 {
t.Errorf("a deliberate empty list was ignored: %+v", n.Weights)
}
}
func TestANewlyFetchedModelAppearsWithoutARestart(t *testing.T) {
// The other direction of the same bug: a model fetched after registration
// was invisible until the agent restarted, so placement could not prefer
// the node that had just spent ten minutes downloading it.
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1",
WeightsChanged: true,
Weights: []*llmgwv1.LocalWeights{
{Path: "/models/new.gguf", SizeBytes: 4 << 30},
},
})
n, _ := s.inv.Get("n1")
if !n.HasWeights("/models/new.gguf") {
t.Errorf("a model fetched after registration did not appear: %+v", n.Weights)
}
}
func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) {
s := newTestService()
before := inventory.Node{
ID: "n1",
State: inventory.Ready,
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
Storage: []discovery.Storage{{Path: "/models", FreeBytes: 7, TotalBytes: 9}},
}
nodeWith(s, before)
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
n, _ := s.inv.Get("n1")
if len(n.Weights) != 1 {
t.Errorf("weights changed: %+v", n.Weights)
}
if free, _ := n.LocalDisk(); free != 7 {
t.Errorf("disk changed: %d", free)
}
}
func TestAHeartbeatCarriesMemoryAddedToARunningNode(t *testing.T) {
// Read once at registration, a node that was given 32 GB went on looking
// like the 7 GB machine it had been, and every model that needed the
// difference stayed unplaced until somebody restarted its agent.
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 32 * 1024})
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
t.Errorf("memory is %d MB, expected the 32 GB the node now has", n.MemoryTotalMB)
}
// Zero is an agent that could not read it, not a node that lost its RAM.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
t.Errorf("a heartbeat without memory erased it: %d", n.MemoryTotalMB)
}
}
func TestAHeartbeatCarriesAnEngineInstalledAfterRegistration(t *testing.T) {
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready,
EngineVersions: map[string]string{"llamacpp": "unknown"}})
// Unchanged engines are not resent, and silence must not clear them.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 1024})
if n, _ := s.inv.Get("n1"); len(n.EngineVersions) != 1 {
t.Fatalf("engines changed without being reported: %v", n.EngineVersions)
}
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
NodeId: "n1", EnginesChanged: true,
EngineVersions: map[string]string{"llamacpp": "unknown", "vllm": "unknown"},
})
n, _ := s.inv.Get("n1")
if _, ok := n.EngineVersions["vllm"]; !ok {
t.Errorf("the newly installed engine is not known: %v", n.EngineVersions)
}
}
func TestALogRequestIsAnsweredByTheNodeItWasSentTo(t *testing.T) {
s := newTestService()
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
s.conns["n1"] = c
type result struct {
lines, sources []string
err error
}
done := make(chan result, 1)
go func() {
l, src, err := s.Logs(t.Context(), "n1", "agent", 10)
done <- result{l, src, err}
}()
req := (<-c.send).GetLogs()
if req.GetSource() != "agent" || req.GetLines() != 10 {
t.Fatalf("sent %v", req)
}
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Logs{Logs: &llmgwv1.LogsReply{
RequestId: req.GetRequestId(), Lines: []string{"hello"}, Sources: []string{"agent", "shard-1"},
}}})
r := <-done
if r.err != nil || len(r.lines) != 1 || r.lines[0] != "hello" || len(r.sources) != 2 {
t.Errorf("got %+v", r)
}
if _, _, err := s.Logs(t.Context(), "nobody", "agent", 10); err == nil {
t.Error("a node with no stream answered")
}
}