Know where the weights are, without owning the storage
CI / test (push) Successful in 1m32s

Storage will be different at every installation — Lustre, NFS over RDMA, an
image exported with SRP or NVMe-oF, local NVMe, an rsync — so llmgw observes
it rather than managing it. An agent told which directories to look in
reports what it can see; how the files got there is nobody's business here,
and a node told nothing reports nothing and behaves exactly as every node
did before this existed.

That fact buys two things. Placement prefers a node that already holds the
model over one that must first move tens of gigabytes, as a preference and
never a filter: fetching them is a valid answer, and refusing to place would
turn a slow start into no start at all. And ctl status shows what each
fabric holds.

Per fabric because fabrics are not connected to one another. Weights in
ib-a are not weights in ib-b — the only path between them is the management
network — so a model seeded into one fabric leaves the rest of the cluster
unable to serve it without a slow copy, and nothing said so before now. One
shared filer counts the same as twenty local copies, because "can this
fabric serve it" is the question being asked.

Three distinctions the tests hold to, because each is a way to get this
quietly wrong. A node that was never asked to look must rank with one that
looked and has nothing, or enabling the feature on a single node would
drain every other. An empty directory is not weights, because a
half-finished download looks exactly like one and would send placement to a
node whose engine then fails to load. And local storage is never mistaken
for shared: a filer read as local costs an unnecessary copy, while local
read as shared would have the cluster believe a model is everywhere when it
is on one node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 08:21:16 +02:00
1 parent bd134a7ab9
commit c13c567cc5
18 files changed
+782 -63

No files matched your search

+9 -3
View File
@@ -59,9 +59,12 @@ type NodeView struct {
// MemoryGB is the machine's RAM, 0 when the agent did not report it. It is
// what decides whether a CPU profile can be placed here, so it belongs
// where someone looks when a model will not schedule.
MemoryGB int `json:"memory_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
MemoryGB int `json:"memory_gb,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
Models []string `json:"models_on_disk,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -163,6 +166,9 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
seen := map[string]struct{}{}
for _, g := range n.GPUs {
if _, dup := seen[g.Class()]; !dup {
+29 -1
View File
@@ -56,6 +56,16 @@ func (c ClassUse) Free() int { return c.Total - c.Used - c.Unhealthy }
type FabricUse struct {
Fabric string `json:"fabric"`
Nodes int `json:"nodes"`
// ModelsPresent is how many nodes here hold weights at all, and Models how
// many distinct models they hold between them.
//
// Per fabric because that is the unit at which the answer is actionable.
// Fabrics are not connected to one another, so weights in one are not
// weights in another: a model seeded into a single fabric leaves the rest
// of the cluster unable to serve it without first moving tens of gigabytes
// over the management network, and nothing said so.
NodesWithWeights int `json:"nodes_with_weights"`
Models int `json:"models_present"`
// LargestFreeBlock is the most nodes on this fabric that are entirely free,
// which is the number that decides whether a multi-node replica can land.
LargestFreeBlock int `json:"largest_free_block"`
@@ -134,6 +144,9 @@ func (a *API) Summary() Summary {
byFabric[f] = &FabricUse{Fabric: f}
}
byFabric[f].Nodes++
if len(n.Models) > 0 {
byFabric[f].NodesWithWeights++
}
// Entirely free, which is what a multi-node replica needs: a node
// with one card in use cannot contribute a full-width shard.
if n.State == "ready" && n.GPUFree == n.GPUTotal && n.GPUTotal > 0 {
@@ -141,7 +154,22 @@ func (a *API) Summary() Summary {
}
}
}
for _, f := range byFabric {
// Distinct models per fabric, counted across its nodes: one copy on a
// shared filer and twenty local copies are the same answer to "can this
// fabric serve it".
perFabricModels := map[string]map[string]struct{}{}
for _, n := range nodes {
for _, f := range n.Fabrics {
if perFabricModels[f] == nil {
perFabricModels[f] = map[string]struct{}{}
}
for _, m := range n.Models {
perFabricModels[f][m] = struct{}{}
}
}
}
for name, f := range byFabric {
f.Models = len(perFabricModels[name])
s.Fabrics = append(s.Fabrics, *f)
}
sort.Slice(s.Fabrics, func(i, j int) bool { return s.Fabrics[i].Fabric < s.Fabrics[j].Fabric })
+44
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -109,3 +110,46 @@ func TestSummaryNamesModelsThatCannotBePlaced(t *testing.T) {
t.Error("no model was counted short")
}
}
func TestWeightsAreCountedPerFabric(t *testing.T) {
// Fabrics are not connected, so weights in one are not weights in another.
// A model seeded into a single fabric leaves the rest of the cluster unable
// to serve it without first moving tens of gigabytes over the management
// network, and nothing said so.
a1 := gpuNode("a1", 4, "ib-a")
a1.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}}
a2 := gpuNode("a2", 4, "ib-a")
a2.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}}
b1 := gpuNode("b1", 4, "ib-b") // the same fabric capacity, none of the weights
api, _ := setup(t, testModels(), a1, a2, b1)
s := api.Summary()
byName := map[string]FabricUse{}
for _, f := range s.Fabrics {
byName[f.Fabric] = f
}
if got := byName["ib-a"]; got.Models != 1 || got.NodesWithWeights != 2 {
t.Errorf("ib-a = %+v, want one model on two nodes", got)
}
if got := byName["ib-b"]; got.Models != 0 || got.NodesWithWeights != 0 {
t.Errorf("ib-b = %+v; it holds nothing and should say so", got)
}
}
func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) {
// A filer mounted on every node and a local copy on every node are the same
// answer to "can this fabric serve it", which is the question being asked.
var nodes []inventory.Node
for _, id := range []string{"n1", "n2", "n3"} {
n := gpuNode(id, 4, "ib-a")
n.Weights = []discovery.LocalWeights{{Path: "/mnt/shared/llama.gguf", Shared: true}}
nodes = append(nodes, n)
}
api, _ := setup(t, testModels(), nodes...)
f := api.Summary().Fabrics[0]
if f.Models != 1 {
t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models)
}
}