CI / test (push) Successful in 1m46s
Found by answering "where can we see this live". `ctl weights` was offering a file for deletion that I had deleted two days earlier, and the node's disk figure was the one it had when its agent started. Discover() is called at registration and nowhere else, so both were frozen. Disk is a fact about a moment, not about a machine. Sampled once, the figure is wrong on exactly the node worth watching — the one busy filling up — and it stays wrong until that agent restarts. The heartbeat already refreshes GPU memory for the same reason, with a comment about noticing a foreign process "after registration"; disk belonged beside it and I did not put it there. The stale list cut both ways. A model fetched after registration was invisible until a restart, so placement could not prefer the node that had just spent ten minutes downloading it, and the disk check could not see the space it used. Storage goes in every heartbeat, being a few numbers per filesystem. The file list goes only when a fingerprint of paths and sizes changes, with a flag saying the list is meant — without that, an empty list would be ambiguous between "this node now holds nothing" and "nothing to report", and the second reading would empty a node's inventory five seconds after it registered. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
115 lines
3.7 KiB
Go
115 lines
3.7 KiB
Go
package agentsvc
|
|
|
|
import (
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) }
|
|
|
|
func newTestService() *Service {
|
|
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{}}
|
|
}
|
|
|
|
func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) {
|
|
// It was sampled once, at registration, and never again. Found on the live
|
|
// cluster: an agent running for two days still reported the disk it had
|
|
// when it started, and a file deleted in between was still being offered
|
|
// for deletion. Disk is a fact about a moment, and the node whose figure
|
|
// matters most is the one busy filling up.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Storage: []discovery.Storage{
|
|
{Path: "/models", FreeBytes: 200 << 30, TotalBytes: 400 << 30},
|
|
},
|
|
})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
Storage: []*llmgwv1.Storage{
|
|
{Path: "/models", FreeBytes: 5 << 30, TotalBytes: 400 << 30},
|
|
},
|
|
})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
free, total := n.LocalDisk()
|
|
if free != 5<<30 || total != 400<<30 {
|
|
t.Errorf("disk is %d of %d, want the figure the heartbeat carried", free, total)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyWeightListOnlyCountsWhenTheAgentMeansIt(t *testing.T) {
|
|
// Most heartbeats carry no file list because nothing changed. Treating
|
|
// that as "this node holds nothing" would empty a node's inventory five
|
|
// seconds after it registered, and placement would stop preferring the
|
|
// node that actually has the model.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
|
|
})
|
|
|
|
// Silence about weights: the list must survive.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
Storage: []*llmgwv1.Storage{{Path: "/models", FreeBytes: 1, TotalBytes: 2}},
|
|
})
|
|
if n, _ := s.inv.Get("n1"); len(n.Weights) != 1 {
|
|
t.Fatalf("a heartbeat with no weight list emptied the inventory: %+v", n.Weights)
|
|
}
|
|
|
|
// And when the agent says it means an empty list, it is taken.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", WeightsChanged: true})
|
|
if n, _ := s.inv.Get("n1"); len(n.Weights) != 0 {
|
|
t.Errorf("a deliberate empty list was ignored: %+v", n.Weights)
|
|
}
|
|
}
|
|
|
|
func TestANewlyFetchedModelAppearsWithoutARestart(t *testing.T) {
|
|
// The other direction of the same bug: a model fetched after registration
|
|
// was invisible until the agent restarted, so placement could not prefer
|
|
// the node that had just spent ten minutes downloading it.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
WeightsChanged: true,
|
|
Weights: []*llmgwv1.LocalWeights{
|
|
{Path: "/models/new.gguf", SizeBytes: 4 << 30},
|
|
},
|
|
})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
if !n.HasWeights("/models/new.gguf") {
|
|
t.Errorf("a model fetched after registration did not appear: %+v", n.Weights)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) {
|
|
s := newTestService()
|
|
before := inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
|
|
Storage: []discovery.Storage{{Path: "/models", FreeBytes: 7, TotalBytes: 9}},
|
|
}
|
|
nodeWith(s, before)
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
if len(n.Weights) != 1 {
|
|
t.Errorf("weights changed: %+v", n.Weights)
|
|
}
|
|
if free, _ := n.LocalDisk(); free != 7 {
|
|
t.Errorf("disk changed: %d", free)
|
|
}
|
|
}
|