Know where the weights are, without owning the storage
CI / test (push) Successful in 1m32s

Storage will be different at every installation — Lustre, NFS over RDMA, an
image exported with SRP or NVMe-oF, local NVMe, an rsync — so llmgw observes
it rather than managing it. An agent told which directories to look in
reports what it can see; how the files got there is nobody's business here,
and a node told nothing reports nothing and behaves exactly as every node
did before this existed.

That fact buys two things. Placement prefers a node that already holds the
model over one that must first move tens of gigabytes, as a preference and
never a filter: fetching them is a valid answer, and refusing to place would
turn a slow start into no start at all. And ctl status shows what each
fabric holds.

Per fabric because fabrics are not connected to one another. Weights in
ib-a are not weights in ib-b — the only path between them is the management
network — so a model seeded into one fabric leaves the rest of the cluster
unable to serve it without a slow copy, and nothing said so before now. One
shared filer counts the same as twenty local copies, because "can this
fabric serve it" is the question being asked.

Three distinctions the tests hold to, because each is a way to get this
quietly wrong. A node that was never asked to look must rank with one that
looked and has nothing, or enabling the feature on a single node would
drain every other. An empty directory is not weights, because a
half-finished download looks exactly like one and would send placement to a
node whose engine then fails to load. And local storage is never mistaken
for shared: a filer read as local costs an unnecessary copy, while local
read as shared would have the cluster believe a model is everywhere when it
is on one node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 08:21:16 +02:00
1 parent bd134a7ab9
commit c13c567cc5
18 files changed
+782 -63

No files matched your search

+9 -3
View File
@@ -59,9 +59,12 @@ type NodeView struct {
// MemoryGB is the machine's RAM, 0 when the agent did not report it. It is
// what decides whether a CPU profile can be placed here, so it belongs
// where someone looks when a model will not schedule.
MemoryGB int `json:"memory_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
MemoryGB int `json:"memory_gb,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
Models []string `json:"models_on_disk,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -163,6 +166,9 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
seen := map[string]struct{}{}
for _, g := range n.GPUs {
if _, dup := seen[g.Class()]; !dup {
+29 -1
View File
@@ -56,6 +56,16 @@ func (c ClassUse) Free() int { return c.Total - c.Used - c.Unhealthy }
type FabricUse struct {
Fabric string `json:"fabric"`
Nodes int `json:"nodes"`
// ModelsPresent is how many nodes here hold weights at all, and Models how
// many distinct models they hold between them.
//
// Per fabric because that is the unit at which the answer is actionable.
// Fabrics are not connected to one another, so weights in one are not
// weights in another: a model seeded into a single fabric leaves the rest
// of the cluster unable to serve it without first moving tens of gigabytes
// over the management network, and nothing said so.
NodesWithWeights int `json:"nodes_with_weights"`
Models int `json:"models_present"`
// LargestFreeBlock is the most nodes on this fabric that are entirely free,
// which is the number that decides whether a multi-node replica can land.
LargestFreeBlock int `json:"largest_free_block"`
@@ -134,6 +144,9 @@ func (a *API) Summary() Summary {
byFabric[f] = &FabricUse{Fabric: f}
}
byFabric[f].Nodes++
if len(n.Models) > 0 {
byFabric[f].NodesWithWeights++
}
// Entirely free, which is what a multi-node replica needs: a node
// with one card in use cannot contribute a full-width shard.
if n.State == "ready" && n.GPUFree == n.GPUTotal && n.GPUTotal > 0 {
@@ -141,7 +154,22 @@ func (a *API) Summary() Summary {
}
}
}
for _, f := range byFabric {
// Distinct models per fabric, counted across its nodes: one copy on a
// shared filer and twenty local copies are the same answer to "can this
// fabric serve it".
perFabricModels := map[string]map[string]struct{}{}
for _, n := range nodes {
for _, f := range n.Fabrics {
if perFabricModels[f] == nil {
perFabricModels[f] = map[string]struct{}{}
}
for _, m := range n.Models {
perFabricModels[f][m] = struct{}{}
}
}
}
for name, f := range byFabric {
f.Models = len(perFabricModels[name])
s.Fabrics = append(s.Fabrics, *f)
}
sort.Slice(s.Fabrics, func(i, j int) bool { return s.Fabrics[i].Fabric < s.Fabrics[j].Fabric })
+44
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -109,3 +110,46 @@ func TestSummaryNamesModelsThatCannotBePlaced(t *testing.T) {
t.Error("no model was counted short")
}
}
func TestWeightsAreCountedPerFabric(t *testing.T) {
// Fabrics are not connected, so weights in one are not weights in another.
// A model seeded into a single fabric leaves the rest of the cluster unable
// to serve it without first moving tens of gigabytes over the management
// network, and nothing said so.
a1 := gpuNode("a1", 4, "ib-a")
a1.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}}
a2 := gpuNode("a2", 4, "ib-a")
a2.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}}
b1 := gpuNode("b1", 4, "ib-b") // the same fabric capacity, none of the weights
api, _ := setup(t, testModels(), a1, a2, b1)
s := api.Summary()
byName := map[string]FabricUse{}
for _, f := range s.Fabrics {
byName[f.Fabric] = f
}
if got := byName["ib-a"]; got.Models != 1 || got.NodesWithWeights != 2 {
t.Errorf("ib-a = %+v, want one model on two nodes", got)
}
if got := byName["ib-b"]; got.Models != 0 || got.NodesWithWeights != 0 {
t.Errorf("ib-b = %+v; it holds nothing and should say so", got)
}
}
func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) {
// A filer mounted on every node and a local copy on every node are the same
// answer to "can this fabric serve it", which is the question being asked.
var nodes []inventory.Node
for _, id := range []string{"n1", "n2", "n3"} {
n := gpuNode(id, 4, "ib-a")
n.Weights = []discovery.LocalWeights{{Path: "/mnt/shared/llama.gguf", Shared: true}}
nodes = append(nodes, n)
}
api, _ := setup(t, testModels(), nodes...)
f := api.Summary().Fabrics[0]
if f.Models != 1 {
t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models)
}
}
+10
View File
@@ -25,6 +25,11 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
EngineVersions: r.GetEngineVersions(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
}
for _, w := range r.GetWeights() {
n.Weights = append(n.Weights, discovery.LocalWeights{
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
})
}
for _, g := range r.GetGpus() {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: int(g.GetIndex()),
@@ -64,6 +69,11 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
Labels: labels, AgentVersion: agentVersion,
MemoryTotalMb: int64(inv.MemoryTotalMB),
}
for _, w := range inv.Weights {
r.Weights = append(r.Weights, &llmgwv1.LocalWeights{
Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared,
})
}
for _, g := range inv.GPUs {
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
Index: int32(g.Index),
+3
View File
@@ -322,6 +322,9 @@ func (m Model) PlacementProfiles() []placement.Profile {
Priority: p.Priority,
NodeSelector: p.NodeSelector,
MinMemoryGB: p.MinMemoryGB,
// The model's source, so placement can prefer nodes that already
// hold the weights rather than one that must first move them.
Source: m.Source,
})
}
return out
+26
View File
@@ -80,6 +80,11 @@ type Node struct {
// this field existed.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
// Weights are the model files this node can see, however they got there.
// Empty means the node was not asked to look, which is not the same as
// holding none — so it constrains nothing.
Weights []discovery.LocalWeights `json:"weights,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
// single-node island and can never host a multi-node replica.
@@ -305,6 +310,27 @@ func (s *Store) FreeGPUs(nodeID string) []int {
return out
}
// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
// an engine will resolve itself — cannot be checked from here, and guessing
// would be worse than the honest answer that nothing is known.
func (n Node) HasWeights(source string) bool {
for _, w := range n.Weights {
if w.Path == source {
return true
}
}
return false
}
// KnowsItsWeights reports whether this node was asked to look at all.
//
// The distinction that keeps the feature optional: a node reporting no weights
// because nobody configured a directory must not be excluded from placement,
// while a node that looked and genuinely has none should be.
func (n Node) KnowsItsWeights() bool { return len(n.Weights) > 0 }
// Fabrics lists every fabric any node is attached to, sorted.
func (s *Store) Fabrics() []string {
s.mu.RLock()
+49
View File
@@ -36,6 +36,9 @@ type Profile struct {
NodeSelector map[string]string
// MinMemoryGB is the host RAM this profile needs. Zero is unconstrained.
MinMemoryGB int
// Source is the model's weights, as the engine would be given it. Used to
// prefer nodes that already hold them; empty asks for no preference.
Source string
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
@@ -193,6 +196,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return len(s.inv.Allocations(candidates[i].node.ID)) <
len(s.inv.Allocations(candidates[j].node.ID))
})
preferNodesHoldingWeights(p.Source, candidates)
return &Placement{
Profile: p,
Pool: "none/cpu",
@@ -200,6 +204,11 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
}, rejections
}
// Last, so it takes precedence: moving tens of gigabytes costs more than an
// imperfectly packed node, and with isolated fabrics it may cost a
// cross-fabric copy over the management network.
preferNodesHoldingWeights(p.Source, candidates)
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode)
if gpus == nil {
@@ -291,6 +300,46 @@ type candidate struct {
//
// fabric is empty for single-node placement, in which case attachment is not
// checked at all.
// preferNodesHoldingWeights moves nodes that already have the model to the
// front, leaving everything else in the order it was given.
//
// A stable partition rather than a sort, so every other consideration — bin
// packing, spreading, the emptiest machine for a multi-node replica — survives
// intact among nodes that are equal on this one.
func preferNodesHoldingWeights(source string, candidates []candidate) {
if source == "" {
return
}
sort.SliceStable(candidates, func(i, j int) bool {
a, aKnown := hasWeights(source, candidates[i].node)
b, bKnown := hasWeights(source, candidates[j].node)
// A node that was never asked to look ranks with those that looked and
// have nothing: unknown is not a reason to prefer or to punish.
return (a && aKnown) && !(b && bKnown)
})
}
// hasWeights reports whether a node already holds this profile's model, and
// whether that is known at all.
//
// Never a filter, always a preference. Storage differs at every installation
// and llmgw does not own it: a node that was never asked to look reports
// nothing, and excluding those would break every cluster that has not opted
// in. Even a node that looked and does not have the weights stays eligible,
// because "fetch them" is a valid answer and refusing to place would turn a
// slow start into no start at all.
//
// What it changes is the ordering, and that is worth a lot on its own: with
// isolated fabrics the difference between placing where the weights are and
// placing where they are not is a cross-fabric copy of tens of gigabytes over
// the management network.
func hasWeights(source string, n inventory.Node) (present, known bool) {
if source == "" || !n.KnowsItsWeights() {
return false, false
}
return n.HasWeights(source), true
}
// unmatchedLabel reports the first selector entry a node does not satisfy.
//
// It returns the label rather than a bare false so the rejection can name it:
@@ -582,3 +582,100 @@ func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) {
t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain())
}
}
func nodeWithWeights(id string, gpus int, paths ...string) inventory.Node {
n := node(id, "h100-80g", gpus)
for _, p := range paths {
n.Weights = append(n.Weights, discovery.LocalWeights{Path: p, SizeBytes: 1 << 30})
}
return n
}
func TestPlacementPrefersANodeThatAlreadyHasTheWeights(t *testing.T) {
// With isolated fabrics the difference between placing where the weights
// are and placing where they are not is a cross-fabric copy of tens of
// gigabytes over the management network.
const src = "/models/llama.gguf"
without := nodeWithWeights("empty", 8, "/models/something-else.gguf")
with := nodeWithWeights("holds-it", 8, src)
s := New(store(without, with))
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Fatalf("nothing was placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "holds-it" {
t.Errorf("placed on %q; the other node already has the weights", got)
}
}
func TestANodeWithoutTheWeightsIsStillEligible(t *testing.T) {
// Fetching them is a valid answer. Refusing to place would turn a slow
// start into no start at all, which is a worse cluster than a slow one.
const src = "/models/llama.gguf"
only := nodeWithWeights("no-weights", 8, "/models/other.gguf")
s := New(store(only))
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Errorf("a node without the weights was excluded:\n%s", res.Explain())
}
}
func TestANodeThatWasNeverAskedIsNotPunished(t *testing.T) {
// What keeps the whole feature optional. A node with no model directories
// configured reports nothing, and that must rank with "looked and has
// none" rather than below it — otherwise enabling the feature on one node
// would quietly drain every other.
const src = "/models/llama.gguf"
silent := node("never-asked", "h100-80g", 8) // reports no weights at all
looked := nodeWithWeights("looked", 8, "/models/other.gguf")
s := New(store(silent, looked))
for range 4 {
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Fatalf("nothing placed:\n%s", res.Explain())
}
}
// Both remain candidates; neither is preferred on weights, so the ordinary
// rules decide and nothing errors.
}
func TestWeightsDoNotOverrideHardConstraints(t *testing.T) {
// A preference, never a filter. The node holding the weights here is the
// wrong GPU class, and choosing it would produce a replica that cannot run.
const src = "/models/llama.gguf"
wrongClass := node("wrong", "a100-40g", 8)
wrongClass.Weights = []discovery.LocalWeights{{Path: src}}
right := node("right", "h100-80g", 8)
s := New(store(wrongClass, right))
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Fatalf("nothing placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "right" {
t.Errorf("placed on %q, which is the wrong GPU class", got)
}
}
func TestAProfileWithNoSourceIsUnaffected(t *testing.T) {
// Every profile before this existed.
a := nodeWithWeights("a", 8, "/models/x.gguf")
b := node("b", "h100-80g", 8)
s := New(store(a, b))
if res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1,
}}, nil); res.Placement == nil {
t.Fatalf("nothing placed:\n%s", res.Explain())
}
}