CI / test (push) Successful in 1m47s
Every agent file on the cluster named its own node, its own address, the path to llama-server and two weight directories — different on every node, kept true by hand, and not true: node05 carried "ram: 32g" beside a measured 31 GB, looked for weights in an empty directory while fetching them into another, and the other four never looked in theirs at all, so four nodes reported no disk. Growing a node's RAM would have gone unnoticed, because memory was read once, at registration. The agent now finds these out. Its id is the hostname, which is also the certificate's common name. Its address is the one on the route to the control plane, which the kernel picks for a UDP dial without sending anything. llama-server is looked for on PATH and then where release archives are unpacked. A missing nvidia-smi is a node without NVIDIA cards rather than a failed discovery — which also cut discovery short before InfiniBand on every CPU node not told to skip GPUs. Fetched weights land in /var/lib/llmgw-weights by default, and wherever they land is always looked in. Memory is read on every heartbeat, and installed engines are probed again each minute and sent when they change, so a VM given more RAM or a node given vLLM becomes a candidate without anyone restarting its agent. The control plane logs both changes, since "the node now qualifies" is the answer to why a model placed an hour after it was asked for. What is written still wins. The agent file shrinks to the control plane's addresses and the TLS files, the same on every node; onboarding writes only the overrides it was given. What cannot be found stays configuration: which fabric an adapter is on, because OpenSM's default prefix is the same on all of them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
158 lines
5.4 KiB
Go
158 lines
5.4 KiB
Go
package agentsvc
|
|
|
|
import (
|
|
"log/slog"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) }
|
|
|
|
func newTestService() *Service {
|
|
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{},
|
|
log: slog.New(slog.DiscardHandler)}
|
|
}
|
|
|
|
func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) {
|
|
// It was sampled once, at registration, and never again. Found on the live
|
|
// cluster: an agent running for two days still reported the disk it had
|
|
// when it started, and a file deleted in between was still being offered
|
|
// for deletion. Disk is a fact about a moment, and the node whose figure
|
|
// matters most is the one busy filling up.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Storage: []discovery.Storage{
|
|
{Path: "/models", FreeBytes: 200 << 30, TotalBytes: 400 << 30},
|
|
},
|
|
})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
Storage: []*llmgwv1.Storage{
|
|
{Path: "/models", FreeBytes: 5 << 30, TotalBytes: 400 << 30},
|
|
},
|
|
})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
free, total := n.LocalDisk()
|
|
if free != 5<<30 || total != 400<<30 {
|
|
t.Errorf("disk is %d of %d, want the figure the heartbeat carried", free, total)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyWeightListOnlyCountsWhenTheAgentMeansIt(t *testing.T) {
|
|
// Most heartbeats carry no file list because nothing changed. Treating
|
|
// that as "this node holds nothing" would empty a node's inventory five
|
|
// seconds after it registered, and placement would stop preferring the
|
|
// node that actually has the model.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
|
|
})
|
|
|
|
// Silence about weights: the list must survive.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
Storage: []*llmgwv1.Storage{{Path: "/models", FreeBytes: 1, TotalBytes: 2}},
|
|
})
|
|
if n, _ := s.inv.Get("n1"); len(n.Weights) != 1 {
|
|
t.Fatalf("a heartbeat with no weight list emptied the inventory: %+v", n.Weights)
|
|
}
|
|
|
|
// And when the agent says it means an empty list, it is taken.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", WeightsChanged: true})
|
|
if n, _ := s.inv.Get("n1"); len(n.Weights) != 0 {
|
|
t.Errorf("a deliberate empty list was ignored: %+v", n.Weights)
|
|
}
|
|
}
|
|
|
|
func TestANewlyFetchedModelAppearsWithoutARestart(t *testing.T) {
|
|
// The other direction of the same bug: a model fetched after registration
|
|
// was invisible until the agent restarted, so placement could not prefer
|
|
// the node that had just spent ten minutes downloading it.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1",
|
|
WeightsChanged: true,
|
|
Weights: []*llmgwv1.LocalWeights{
|
|
{Path: "/models/new.gguf", SizeBytes: 4 << 30},
|
|
},
|
|
})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
if !n.HasWeights("/models/new.gguf") {
|
|
t.Errorf("a model fetched after registration did not appear: %+v", n.Weights)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) {
|
|
s := newTestService()
|
|
before := inventory.Node{
|
|
ID: "n1",
|
|
State: inventory.Ready,
|
|
Weights: []discovery.LocalWeights{{Path: "/models/a.gguf", SizeBytes: 100}},
|
|
Storage: []discovery.Storage{{Path: "/models", FreeBytes: 7, TotalBytes: 9}},
|
|
}
|
|
nodeWith(s, before)
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
|
|
|
n, _ := s.inv.Get("n1")
|
|
if len(n.Weights) != 1 {
|
|
t.Errorf("weights changed: %+v", n.Weights)
|
|
}
|
|
if free, _ := n.LocalDisk(); free != 7 {
|
|
t.Errorf("disk changed: %d", free)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatCarriesMemoryAddedToARunningNode(t *testing.T) {
|
|
// Read once at registration, a node that was given 32 GB went on looking
|
|
// like the 7 GB machine it had been, and every model that needed the
|
|
// difference stayed unplaced until somebody restarted its agent.
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024})
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 32 * 1024})
|
|
|
|
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
|
|
t.Errorf("memory is %d MB, expected the 32 GB the node now has", n.MemoryTotalMB)
|
|
}
|
|
|
|
// Zero is an agent that could not read it, not a node that lost its RAM.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
|
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
|
|
t.Errorf("a heartbeat without memory erased it: %d", n.MemoryTotalMB)
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatCarriesAnEngineInstalledAfterRegistration(t *testing.T) {
|
|
s := newTestService()
|
|
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready,
|
|
EngineVersions: map[string]string{"llamacpp": "unknown"}})
|
|
|
|
// Unchanged engines are not resent, and silence must not clear them.
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 1024})
|
|
if n, _ := s.inv.Get("n1"); len(n.EngineVersions) != 1 {
|
|
t.Fatalf("engines changed without being reported: %v", n.EngineVersions)
|
|
}
|
|
|
|
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
|
NodeId: "n1", EnginesChanged: true,
|
|
EngineVersions: map[string]string{"llamacpp": "unknown", "vllm": "unknown"},
|
|
})
|
|
n, _ := s.inv.Get("n1")
|
|
if _, ok := n.EngineVersions["vllm"]; !ok {
|
|
t.Errorf("the newly installed engine is not known: %v", n.EngineVersions)
|
|
}
|
|
}
|