Reserve CPU cores and require memory for CPU models
A profile's cpu_cores is reserved out of its node's CPUs by placement, as memory is, and becomes the engine's thread count. Agents report their CPU count in registration and heartbeats. New CPU models must declare memory; model views show reserved against measured memory with warnings, and ctl model resources sets both for a stored model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
26151ec25a
commit
990d82d801
31 files changed
+610
-53
No files matched your search
@@ -57,6 +57,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models", a.addModel)
|
||||
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
|
||||
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
|
||||
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
|
||||
@@ -119,6 +120,8 @@ type NodeView struct {
|
||||
// what decides whether a CPU profile can be placed here, so it belongs
|
||||
// where someone looks when a model will not schedule.
|
||||
MemoryGB int `json:"memory_gb,omitempty"`
|
||||
// CPUCount is the machine's CPUs, 0 when the agent did not report them.
|
||||
CPUCount int `json:"cpu_count,omitempty"`
|
||||
// Engines are the inference engines installed here, as the agent found
|
||||
// them. Absent means the agent is older than the probe and said nothing,
|
||||
// which is not the same as having none — placement treats it that way too,
|
||||
@@ -270,6 +273,15 @@ type ModelView struct {
|
||||
Unplaceable bool `json:"unplaceable,omitempty"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
Profiles []string `json:"profiles"`
|
||||
// MemoryGB and CPUCores are what one replica reserves, from its first
|
||||
// profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
|
||||
// when none is measured.
|
||||
MemoryGB int `json:"memory_gb,omitempty"`
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"`
|
||||
// Warnings are what makes its reservation wrong: nothing declared where
|
||||
// placement needs a figure, or less declared than it uses.
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
@@ -396,8 +408,8 @@ func (a *API) Nodes() []NodeView {
|
||||
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
|
||||
State: n.State.String(), Labels: n.Labels,
|
||||
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
|
||||
MemoryGB: n.MemoryTotalMB / 1024,
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||
MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount,
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||
Replicas: replicasOn[n.ID],
|
||||
}
|
||||
if free, total := n.LocalDisk(); total > 0 {
|
||||
@@ -497,6 +509,15 @@ func (a *API) Models() []ModelView {
|
||||
for _, h := range a.ctl.Reconciler().ModelHealth() {
|
||||
health[h.Model] = h
|
||||
}
|
||||
// The most one replica of each model uses, summed over its shards.
|
||||
measured := map[string]int64{}
|
||||
for _, rep := range a.Replicas("") {
|
||||
var b int64
|
||||
for _, sh := range rep.Shards {
|
||||
b += sh.MemoryBytes
|
||||
}
|
||||
measured[rep.Model] = max(measured[rep.Model], b)
|
||||
}
|
||||
out := make([]ModelView, 0)
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
h := health[m.Name]
|
||||
@@ -511,6 +532,11 @@ func (a *API) Models() []ModelView {
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
}
|
||||
if len(m.Profiles) > 0 {
|
||||
v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
|
||||
}
|
||||
v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
|
||||
v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
@@ -581,6 +607,15 @@ type Models interface {
|
||||
Add(ctx context.Context, m catalog.Model, by string) error
|
||||
Remove(ctx context.Context, name string) error
|
||||
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
type ResourcesRequest struct {
|
||||
// Profile is one profile's name; empty is every profile.
|
||||
Profile string `json:"profile,omitempty"`
|
||||
MemoryGB int `json:"memory_gb"`
|
||||
CPUCores int `json:"cpu_cores"`
|
||||
}
|
||||
|
||||
// Usage reports what each group of the tenancy tree spent. The store
|
||||
@@ -826,6 +861,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// setResources sets a model's reserved memory and cores. Placed replicas keep
|
||||
// what they were started with until they are restarted.
|
||||
func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req ResourcesRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, m)
|
||||
}
|
||||
|
||||
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
@@ -1307,3 +1362,40 @@ func actorOf(r *http.Request) string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// NodeCPUs is the CPU counts of ready nodes with at least this much memory:
|
||||
// where a replica needing it could go.
|
||||
func (a *API) NodeCPUs(minMemoryGB int) []int {
|
||||
var out []int
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
if n.State.String() != "ready" || n.CPUCount == 0 {
|
||||
continue
|
||||
}
|
||||
if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB {
|
||||
continue
|
||||
}
|
||||
out = append(out, n.CPUCount)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reservationWarnings says what is wrong with what a model reserves.
|
||||
//
|
||||
// Resident memory includes the weights mapped from disk, which the kernel can
|
||||
// take back, so it overstates what a model needs by the part of the weights
|
||||
// not in use. A tenth over is let pass for that; more is a figure too low for
|
||||
// placement to be trusted with.
|
||||
func reservationWarnings(m catalog.Model, measuredGB float64) []string {
|
||||
var out []string
|
||||
for _, p := range m.Profiles {
|
||||
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
||||
out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+
|
||||
"(llmgw ctl model resources %s --memory GB)", p.Name, m.Name))
|
||||
}
|
||||
}
|
||||
if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 {
|
||||
out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory",
|
||||
measuredGB, m.Profiles[0].MinMemoryGB))
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user