Open a held model to its replicas, nodes and room for more
The Held tab's model name opens a card: each replica with its node, state, age, CPU and memory, the room for more found by dry-running the scheduler on a copy of the inventory, and a form for the always-running and under-load replica counts. An always-running count placement cannot meet is refused with the reason. Headroom is also on the admin API. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2ae12ae1fe
commit
d7f1c1d545
11 files changed
+683
-4
No files matched your search
@@ -15,6 +15,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
@@ -91,6 +92,15 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
v, ok := a.Headroom(r.PathValue("name"), limit)
|
||||
if !ok {
|
||||
http.Error(w, "no such model", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
}
|
||||
|
||||
// --- responses ---
|
||||
@@ -336,6 +346,22 @@ type ExplainView struct {
|
||||
Rejections []RejectionView `json:"rejections,omitempty"`
|
||||
}
|
||||
|
||||
// HeadroomView is how many more replicas of a model fit now, and where.
|
||||
type HeadroomView struct {
|
||||
Model string `json:"model"`
|
||||
// Current is the replicas holding capacity: starting or ready.
|
||||
Current int `json:"current"`
|
||||
// More is how many more placement would find room for now.
|
||||
More int `json:"more"`
|
||||
// Nodes is where each of those would go.
|
||||
Nodes [][]string `json:"nodes,omitempty"`
|
||||
// Limited is set when More stopped at the limit asked rather than at the
|
||||
// cluster's edge: there may be room for more.
|
||||
Limited bool `json:"limited,omitempty"`
|
||||
// Blocked says why one more does not fit, a line per node considered.
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
}
|
||||
|
||||
// RejectionView is one rejected candidate.
|
||||
type RejectionView struct {
|
||||
Profile string `json:"profile"`
|
||||
@@ -836,6 +862,20 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
||||
// It does not check that the model exists; the handler above does that so it
|
||||
// can answer 404, while the dashboard only ever asks about models it just read
|
||||
// out of the catalog.
|
||||
// Headroom tries placing up to limit more replicas of a model, on a copy of
|
||||
// the inventory, and says how many fit. A limit of zero or less takes 16.
|
||||
func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
|
||||
if limit <= 0 || limit > 64 {
|
||||
limit = 16
|
||||
}
|
||||
h, ok := a.ctl.Reconciler().Headroom(model, limit)
|
||||
if !ok {
|
||||
return HeadroomView{}, false
|
||||
}
|
||||
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
|
||||
Limited: h.Limited, Blocked: h.Blocked}, true
|
||||
}
|
||||
|
||||
func (a *API) Explain(model string) ExplainView {
|
||||
v := ExplainView{Model: model}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
|
||||
@@ -392,6 +392,23 @@ func NewStore() *Store {
|
||||
return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
|
||||
}
|
||||
|
||||
// Clone is a copy to try decisions on without making them: allocations made
|
||||
// on it are its own. Nodes are copied by value; the slices inside them are
|
||||
// shared, and a clone must not write through them.
|
||||
func (s *Store) Clone() *Store {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
c := NewStore()
|
||||
for id, n := range s.nodes {
|
||||
cp := *n
|
||||
c.nodes[id] = &cp
|
||||
}
|
||||
for id, list := range s.allocs {
|
||||
c.allocs[id] = slices.Clone(list)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Upsert records a node's registration, preserving operator-set state such as
|
||||
// a cordon: an agent reconnecting must not silently undo an operator's decision.
|
||||
// RecordSMChange keeps a port's change of master subnet manager, the last
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
)
|
||||
|
||||
// Headroom is how many more replicas of a model the cluster has room for now.
|
||||
type Headroom struct {
|
||||
// Current is the replicas the model has that hold capacity: starting or
|
||||
// ready, not those on their way out.
|
||||
Current int
|
||||
// More is how many more could be placed now, at most the limit asked.
|
||||
More int
|
||||
// Nodes is where each of those would go, in order, a replica's nodes
|
||||
// joined when it spans several.
|
||||
Nodes [][]string
|
||||
// Limited is set when More reached the limit asked, so there may be room
|
||||
// for more than that. A CPU profile that declares no memory is always
|
||||
// limited: nothing it asks for runs out.
|
||||
Limited bool
|
||||
// Blocked is why one more than More does not fit, a line per node that
|
||||
// was considered. Empty when Limited.
|
||||
Blocked []string
|
||||
}
|
||||
|
||||
// Headroom tries placing up to limit more replicas of a model on a copy of
|
||||
// the inventory, each one's allocation made on the copy before the next is
|
||||
// tried, exactly as the reconciler would place them one pass at a time.
|
||||
//
|
||||
// The scheduler's own decisions, not an estimate beside them: what the page
|
||||
// offers is then what the next passes will do, and a count that the
|
||||
// arithmetic of a second implementation allowed but placement refuses would
|
||||
// leave a model short with nobody told why.
|
||||
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
if !ok {
|
||||
return Headroom{}, false
|
||||
}
|
||||
var h Headroom
|
||||
var avoid []string
|
||||
r.mu.Lock()
|
||||
for _, rep := range r.replicas {
|
||||
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
|
||||
h.Current++
|
||||
avoid = append(avoid, rep.NodeIDs()...)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
inv := r.inv.Clone()
|
||||
sched := placement.New(inv)
|
||||
for h.More < limit {
|
||||
res := sched.Place(m.PlacementProfiles(), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
// As scaleUp does: spreading is a preference, not a rule.
|
||||
res = sched.Place(m.PlacementProfiles(), nil)
|
||||
}
|
||||
if res.Placement == nil {
|
||||
for _, rej := range res.Rejections {
|
||||
h.Blocked = append(h.Blocked, rej.String())
|
||||
}
|
||||
slices.Sort(h.Blocked)
|
||||
h.Blocked = slices.Compact(h.Blocked)
|
||||
return h, true
|
||||
}
|
||||
mem := 0
|
||||
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
|
||||
mem = prof.MinMemoryGB
|
||||
}
|
||||
var nodes []string
|
||||
for _, a := range res.Placement.Nodes {
|
||||
inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem,
|
||||
})
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
avoid = append(avoid, nodes...)
|
||||
h.Nodes = append(h.Nodes, nodes)
|
||||
h.More++
|
||||
}
|
||||
h.Limited = true
|
||||
return h, true
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func TestHeadroomIsWhatPlacementWouldDoNext(t *testing.T) {
|
||||
// Two 8-GPU nodes and a 4-GPU profile: four replicas in all, one placed.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
|
||||
got, ok := h.r.Headroom("llama", 10)
|
||||
if !ok {
|
||||
t.Fatal("the model was not found")
|
||||
}
|
||||
if got.Current != 1 || got.More != 3 || got.Limited {
|
||||
t.Fatalf("headroom %+v, expected 1 placed and room for 3 more", got)
|
||||
}
|
||||
// Spread first: the next one goes to the node without one.
|
||||
if first := got.Nodes[0][0]; first == h.r.Replicas()[0].NodeIDs()[0] {
|
||||
t.Errorf("the next replica would stack on %s beside the first", first)
|
||||
}
|
||||
if len(got.Blocked) == 0 || !strings.Contains(strings.Join(got.Blocked, "\n"), "GPU") {
|
||||
t.Errorf("no reason given for the fifth: %v", got.Blocked)
|
||||
}
|
||||
// Tried on a copy: nothing was taken from the real inventory.
|
||||
if free := len(h.inv.FreeGPUs("1")) + len(h.inv.FreeGPUs("2")); free != 12 {
|
||||
t.Errorf("%d GPUs free after asking, expected the 12 the one replica leaves", free)
|
||||
}
|
||||
|
||||
capped, _ := h.r.Headroom("llama", 2)
|
||||
if capped.More != 2 || !capped.Limited || capped.Blocked != nil {
|
||||
t.Errorf("asked for at most 2: %+v", capped)
|
||||
}
|
||||
if _, ok := h.r.Headroom("nope", 5); ok {
|
||||
t.Error("an unknown model has headroom")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) {
|
||||
cpu := func(id string) inventory.Node {
|
||||
return inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready, MemoryTotalMB: 16 * 1024}
|
||||
}
|
||||
m := catalog.Model{Name: "small", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6}}}
|
||||
h := newHarness(t, m, cpu("a"), cpu("b"))
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
// 16 GB each at 6 GB a replica: two per node, one of them taken.
|
||||
got, _ := h.r.Headroom("small", 10)
|
||||
if got.Current != 1 || got.More != 3 {
|
||||
t.Errorf("headroom %+v, expected room for 3 more", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user