Open a held model to its replicas, nodes and room for more
Release / build (push) Successful in 1m6s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m48s

The Held tab's model name opens a card: each replica with its node, state,
age, CPU and memory, the room for more found by dry-running the scheduler
on a copy of the inventory, and a form for the always-running and
under-load replica counts. An always-running count placement cannot meet
is refused with the reason. Headroom is also on the admin API.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 10:19:31 +02:00
1 parent 2ae12ae1fe
commit d7f1c1d545
11 files changed
+683 -4

No files matched your search

+40
View File
@@ -15,6 +15,7 @@ import (
"io"
"net/http"
"slices"
"strconv"
"strings"
"time"
"unicode"
@@ -91,6 +92,15 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
v, ok := a.Headroom(r.PathValue("name"), limit)
if !ok {
http.Error(w, "no such model", http.StatusNotFound)
return
}
writeJSON(w, http.StatusOK, v)
})
}
// --- responses ---
@@ -336,6 +346,22 @@ type ExplainView struct {
Rejections []RejectionView `json:"rejections,omitempty"`
}
// HeadroomView is how many more replicas of a model fit now, and where.
type HeadroomView struct {
Model string `json:"model"`
// Current is the replicas holding capacity: starting or ready.
Current int `json:"current"`
// More is how many more placement would find room for now.
More int `json:"more"`
// Nodes is where each of those would go.
Nodes [][]string `json:"nodes,omitempty"`
// Limited is set when More stopped at the limit asked rather than at the
// cluster's edge: there may be room for more.
Limited bool `json:"limited,omitempty"`
// Blocked says why one more does not fit, a line per node considered.
Blocked []string `json:"blocked,omitempty"`
}
// RejectionView is one rejected candidate.
type RejectionView struct {
Profile string `json:"profile"`
@@ -836,6 +862,20 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
// It does not check that the model exists; the handler above does that so it
// can answer 404, while the dashboard only ever asks about models it just read
// out of the catalog.
// Headroom tries placing up to limit more replicas of a model, on a copy of
// the inventory, and says how many fit. A limit of zero or less takes 16.
func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
if limit <= 0 || limit > 64 {
limit = 16
}
h, ok := a.ctl.Reconciler().Headroom(model, limit)
if !ok {
return HeadroomView{}, false
}
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
Limited: h.Limited, Blocked: h.Blocked}, true
}
func (a *API) Explain(model string) ExplainView {
v := ExplainView{Model: model}
for _, rep := range a.ctl.Reconciler().Replicas() {
+17
View File
@@ -392,6 +392,23 @@ func NewStore() *Store {
return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
}
// Clone is a copy to try decisions on without making them: allocations made
// on it are its own. Nodes are copied by value; the slices inside them are
// shared, and a clone must not write through them.
func (s *Store) Clone() *Store {
s.mu.RLock()
defer s.mu.RUnlock()
c := NewStore()
for id, n := range s.nodes {
cp := *n
c.nodes[id] = &cp
}
for id, list := range s.allocs {
c.allocs[id] = slices.Clone(list)
}
return c
}
// Upsert records a node's registration, preserving operator-set state such as
// a cordon: an agent reconnecting must not silently undo an operator's decision.
// RecordSMChange keeps a port's change of master subnet manager, the last
+86
View File
@@ -0,0 +1,86 @@
package reconciler
import (
"slices"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// Headroom is how many more replicas of a model the cluster has room for now.
type Headroom struct {
// Current is the replicas the model has that hold capacity: starting or
// ready, not those on their way out.
Current int
// More is how many more could be placed now, at most the limit asked.
More int
// Nodes is where each of those would go, in order, a replica's nodes
// joined when it spans several.
Nodes [][]string
// Limited is set when More reached the limit asked, so there may be room
// for more than that. A CPU profile that declares no memory is always
// limited: nothing it asks for runs out.
Limited bool
// Blocked is why one more than More does not fit, a line per node that
// was considered. Empty when Limited.
Blocked []string
}
// Headroom tries placing up to limit more replicas of a model on a copy of
// the inventory, each one's allocation made on the copy before the next is
// tried, exactly as the reconciler would place them one pass at a time.
//
// The scheduler's own decisions, not an estimate beside them: what the page
// offers is then what the next passes will do, and a count that the
// arithmetic of a second implementation allowed but placement refuses would
// leave a model short with nobody told why.
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
m, ok := r.cat.Load().Resolve(model)
if !ok {
return Headroom{}, false
}
var h Headroom
var avoid []string
r.mu.Lock()
for _, rep := range r.replicas {
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
h.Current++
avoid = append(avoid, rep.NodeIDs()...)
}
}
r.mu.Unlock()
inv := r.inv.Clone()
sched := placement.New(inv)
for h.More < limit {
res := sched.Place(m.PlacementProfiles(), avoid)
if res.Placement == nil && len(avoid) > 0 {
// As scaleUp does: spreading is a preference, not a rule.
res = sched.Place(m.PlacementProfiles(), nil)
}
if res.Placement == nil {
for _, rej := range res.Rejections {
h.Blocked = append(h.Blocked, rej.String())
}
slices.Sort(h.Blocked)
h.Blocked = slices.Compact(h.Blocked)
return h, true
}
mem := 0
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
mem = prof.MinMemoryGB
}
var nodes []string
for _, a := range res.Placement.Nodes {
inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem,
})
nodes = append(nodes, a.NodeID)
}
avoid = append(avoid, nodes...)
h.Nodes = append(h.Nodes, nodes)
h.More++
}
h.Limited = true
return h, true
}
@@ -0,0 +1,59 @@
package reconciler
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestHeadroomIsWhatPlacementWouldDoNext(t *testing.T) {
// Two 8-GPU nodes and a 4-GPU profile: four replicas in all, one placed.
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
got, ok := h.r.Headroom("llama", 10)
if !ok {
t.Fatal("the model was not found")
}
if got.Current != 1 || got.More != 3 || got.Limited {
t.Fatalf("headroom %+v, expected 1 placed and room for 3 more", got)
}
// Spread first: the next one goes to the node without one.
if first := got.Nodes[0][0]; first == h.r.Replicas()[0].NodeIDs()[0] {
t.Errorf("the next replica would stack on %s beside the first", first)
}
if len(got.Blocked) == 0 || !strings.Contains(strings.Join(got.Blocked, "\n"), "GPU") {
t.Errorf("no reason given for the fifth: %v", got.Blocked)
}
// Tried on a copy: nothing was taken from the real inventory.
if free := len(h.inv.FreeGPUs("1")) + len(h.inv.FreeGPUs("2")); free != 12 {
t.Errorf("%d GPUs free after asking, expected the 12 the one replica leaves", free)
}
capped, _ := h.r.Headroom("llama", 2)
if capped.More != 2 || !capped.Limited || capped.Blocked != nil {
t.Errorf("asked for at most 2: %+v", capped)
}
if _, ok := h.r.Headroom("nope", 5); ok {
t.Error("an unknown model has headroom")
}
}
func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) {
cpu := func(id string) inventory.Node {
return inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready, MemoryTotalMB: 16 * 1024}
}
m := catalog.Model{Name: "small", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6}}}
h := newHarness(t, m, cpu("a"), cpu("b"))
h.r.Reconcile(t.Context())
// 16 GB each at 6 GB a replica: two per node, one of them taken.
got, _ := h.r.Headroom("small", 10)
if got.Current != 1 || got.More != 3 {
t.Errorf("headroom %+v, expected room for 3 more", got)
}
}