Reserve CPU cores and require memory for CPU models
Release / build (push) Successful in 1m5s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m47s

A profile's cpu_cores is reserved out of its node's CPUs by placement, as
memory is, and becomes the engine's thread count. Agents report their CPU
count in registration and heartbeats. New CPU models must declare memory;
model views show reserved against measured memory with warnings, and
ctl model resources sets both for a stored model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 17:43:13 +02:00
1 parent 26151ec25a
commit 990d82d801
31 files changed
+610 -53

No files matched your search

+94 -2
View File
@@ -57,6 +57,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models", a.addModel)
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
@@ -119,6 +120,8 @@ type NodeView struct {
// what decides whether a CPU profile can be placed here, so it belongs
// where someone looks when a model will not schedule.
MemoryGB int `json:"memory_gb,omitempty"`
// CPUCount is the machine's CPUs, 0 when the agent did not report them.
CPUCount int `json:"cpu_count,omitempty"`
// Engines are the inference engines installed here, as the agent found
// them. Absent means the agent is older than the probe and said nothing,
// which is not the same as having none — placement treats it that way too,
@@ -270,6 +273,15 @@ type ModelView struct {
Unplaceable bool `json:"unplaceable,omitempty"`
Fallbacks []string `json:"fallbacks,omitempty"`
Profiles []string `json:"profiles"`
// MemoryGB and CPUCores are what one replica reserves, from its first
// profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
// when none is measured.
MemoryGB int `json:"memory_gb,omitempty"`
CPUCores int `json:"cpu_cores,omitempty"`
MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"`
// Warnings are what makes its reservation wrong: nothing declared where
// placement needs a figure, or less declared than it uses.
Warnings []string `json:"warnings,omitempty"`
}
// ReplicaView is a replica as `ctl replicas` shows it.
@@ -396,8 +408,8 @@ func (a *API) Nodes() []NodeView {
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
State: n.State.String(), Labels: n.Labels,
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
MemoryGB: n.MemoryTotalMB / 1024,
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount,
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
Replicas: replicasOn[n.ID],
}
if free, total := n.LocalDisk(); total > 0 {
@@ -497,6 +509,15 @@ func (a *API) Models() []ModelView {
for _, h := range a.ctl.Reconciler().ModelHealth() {
health[h.Model] = h
}
// The most one replica of each model uses, summed over its shards.
measured := map[string]int64{}
for _, rep := range a.Replicas("") {
var b int64
for _, sh := range rep.Shards {
b += sh.MemoryBytes
}
measured[rep.Model] = max(measured[rep.Model], b)
}
out := make([]ModelView, 0)
for _, m := range a.ctl.Catalog().Models() {
h := health[m.Name]
@@ -511,6 +532,11 @@ func (a *API) Models() []ModelView {
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
}
if len(m.Profiles) > 0 {
v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
}
v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
out = append(out, v)
}
return out
@@ -581,6 +607,15 @@ type Models interface {
Add(ctx context.Context, m catalog.Model, by string) error
Remove(ctx context.Context, name string) error
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
}
// ResourcesRequest sets what a model's profiles reserve of a node.
type ResourcesRequest struct {
// Profile is one profile's name; empty is every profile.
Profile string `json:"profile,omitempty"`
MemoryGB int `json:"memory_gb"`
CPUCores int `json:"cpu_cores"`
}
// Usage reports what each group of the tenancy tree spent. The store
@@ -826,6 +861,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
})
}
// setResources sets a model's reserved memory and cores. Placed replicas keep
// what they were started with until they are restarted.
func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req ResourcesRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, m)
}
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
@@ -1307,3 +1362,40 @@ func actorOf(r *http.Request) string {
}
return b.String()
}
// NodeCPUs is the CPU counts of ready nodes with at least this much memory:
// where a replica needing it could go.
func (a *API) NodeCPUs(minMemoryGB int) []int {
var out []int
for _, n := range a.ctl.Inventory().Nodes() {
if n.State.String() != "ready" || n.CPUCount == 0 {
continue
}
if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB {
continue
}
out = append(out, n.CPUCount)
}
return out
}
// reservationWarnings says what is wrong with what a model reserves.
//
// Resident memory includes the weights mapped from disk, which the kernel can
// take back, so it overstates what a model needs by the part of the weights
// not in use. A tenth over is let pass for that; more is a figure too low for
// placement to be trusted with.
func reservationWarnings(m catalog.Model, measuredGB float64) []string {
var out []string
for _, p := range m.Profiles {
if p.CPUOnly() && p.MinMemoryGB <= 0 {
out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+
"(llmgw ctl model resources %s --memory GB)", p.Name, m.Name))
}
}
if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 {
out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory",
measuredGB, m.Profiles[0].MinMemoryGB))
}
return out
}