From 990d82d801f38506aee4205d5973a06285856988 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 17:43:13 +0200 Subject: [PATCH] Reserve CPU cores and require memory for CPU models A profile's cpu_cores is reserved out of its node's CPUs by placement, as memory is, and becomes the engine's thread count. Agents report their CPU count in registration and heartbeats. New CPU models must declare memory; model views show reserved against measured memory with warnings, and ctl model resources sets both for a stored model. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 24 +++++ api/gen/llmgw/v1/agent.pb.go | 52 +++++++++-- api/proto/llmgw/v1/agent.proto | 9 ++ docs/reference.md | 15 +++ internal/agent/agent.go | 2 + internal/agent/engines/engine.go | 4 + internal/agent/engines/llamacpp.go | 7 +- internal/agent/engines/llamacpp_test.go | 12 +++ internal/cli/ctl.go | 38 +++++++- internal/cli/model.go | 41 +++++++++ internal/cli/model_serve.go | 1 + internal/control/adminapi/actor_test.go | 21 +++++ internal/control/adminapi/api.go | 96 +++++++++++++++++++- internal/control/adminapi/api_test.go | 7 ++ internal/control/agentsvc/convert.go | 5 + internal/control/agentsvc/service.go | 9 +- internal/control/catalog/catalog.go | 28 ++++++ internal/control/inventory/inventory.go | 6 ++ internal/control/modelserve/modelserve.go | 12 ++- internal/control/modelsvc/modelsvc.go | 88 +++++++++++++----- internal/control/modelsvc/modelsvc_test.go | 42 ++++++++- internal/control/placement/cpu_test.go | 36 ++++++++ internal/control/placement/placement.go | 24 +++++ internal/control/reconciler/adopt.go | 7 ++ internal/control/reconciler/headroom.go | 6 +- internal/control/reconciler/headroom_test.go | 21 +++++ internal/control/reconciler/reconciler.go | 24 ++++- internal/web/admin_serve.go | 17 ++++ internal/web/static/base.css | 1 + internal/web/templates/admin_mirror.html | 4 +- internal/web/templates/dashboard.html | 4 +- 31 files changed, 610 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddf293..3f593e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,30 @@ APIs change between minors without a deprecation period. Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until there are enough for v0.10.5. +### CPU cores are reserved, and memory has to be declared + +The first step of resource allocation for larger clusters: what a replica +holds of a node is reserved, not assumed. + +- **Cores.** A profile's `cpu_cores` is reserved out of its node's CPUs, as + memory is, and is what its engine computes on (llama.cpp's `--threads`). A + node holds as many CPU replicas as it has cores for, and two on one node no + longer spin against each other. Agents report their CPU count; a node that + does not, and a profile that declares no cores, constrain nothing. +- **Memory is required of new CPU models.** A model with no GPU is all host + memory, and one declaring none looked to placement as if it fitted + everywhere, any number of times. Adding one without `min_memory_gb` is now + refused. Models stored before this still load, and say what is missing. +- **Declared against measured.** Each model shows what one replica reserves + and the most one of its replicas uses — in `ctl models` (RESERVES) and on the + cluster page — with a warning when nothing is declared or when it uses more + than a tenth over what it reserves. +- **Setting them.** `llmgw ctl model resources MODEL --memory GB --cores N` + (and `POST /api/v1/models/{name}/resources`) for a model added at runtime. + `ctl model serve` and the mirror's serve form fill in cores — half the + smallest node the model fits on — beside the memory they already estimate. + Running replicas keep their old figures until restarted. + ### The mirror page in tabs /admin/mirror was one long page: what the mirror holds, the Hugging Face diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index b0ef34a..41adf67 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -534,6 +534,10 @@ type Register struct { // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"` + // cpu_count is the CPUs the agent may run on. Placement reserves cores + // for CPU replicas out of it, the way it reserves memory out of + // memory_total_mb; zero, from an older agent, reserves nothing. + CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"` // imex is the service that lets a multi-node NVLink domain share memory. Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"` // installable_engines are the engines this agent knows how to install @@ -679,6 +683,13 @@ func (x *Register) GetEnginesProbed() bool { return false } +func (x *Register) GetCpuCount() int32 { + if x != nil { + return x.CpuCount + } + return 0 +} + func (x *Register) GetImex() *IMEX { if x != nil { return x.Imex @@ -2017,8 +2028,10 @@ type Heartbeat struct { // gpu_fabric is each GPU's NVLink domain membership as it stands now, sent // once a minute: a fabric manager restarting drops every card out of its // domain, and the node has to stop being offered work that spans it. - GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"` - Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"` + GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"` + Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"` + // cpu_count as in Register, for a VM given more CPUs while it runs. + CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2172,6 +2185,13 @@ func (x *Heartbeat) GetImex() *IMEX { return nil } +func (x *Heartbeat) GetCpuCount() int32 { + if x != nil { + return x.CpuCount + } + return 0 +} + // Provision is the software a node should have: which engines, at which // versions. Nothing more. // @@ -2561,7 +2581,10 @@ type StartShard struct { // api_key is what the engine demands of every request, derived by the // control plane from the replica id. The router sends it; nothing else // that can reach the node's ports has it. - ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"` + ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"` + // cpu_threads is how many CPU threads a shard with no GPU computes on: + // the cores placement reserved for it. Zero leaves it to the agent. + CpuThreads int32 `protobuf:"varint,17,opt,name=cpu_threads,json=cpuThreads,proto3" json:"cpu_threads,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2708,6 +2731,13 @@ func (x *StartShard) GetApiKey() string { return "" } +func (x *StartShard) GetCpuThreads() int32 { + if x != nil { + return x.CpuThreads + } + return 0 +} + type EngineProfile struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -3655,7 +3685,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "fabricTest\x122\n" + "\aupgrade\x18\n" + " \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" + - "\x03msg\"\xe3\x06\n" + + "\x03msg\"\x80\a\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -3671,7 +3701,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" + "\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" + - "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" + + "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\x1b\n" + + "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x12\"\n" + "\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" + "\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x12\x1a\n" + "\bfeatures\x18\x11 \x03(\tR\bfeatures\x1a9\n" + @@ -3811,7 +3842,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06status\x18\x03 \x01(\tR\x06status\x12!\n" + "\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" + "\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" + - "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd9\x06\n" + + "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xf6\x06\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -3831,7 +3862,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x127\n" + "\n" + "gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" + - "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1aA\n" + + "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" + + "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + @@ -3872,7 +3904,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + "\vbytes_total\x18\x03 \x01(\x03R\n" + - "bytesTotal\"\xf2\x03\n" + + "bytesTotal\"\x93\x04\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -3896,7 +3928,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" + "\n" + "profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" + - "\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" + + "\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" + + "\vcpu_threads\x18\x11 \x01(\x05R\n" + + "cpuThreads\"\xfd\x02\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 811f095..cd24d04 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -120,6 +120,10 @@ message Register { // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. bool engines_probed = 14; + // cpu_count is the CPUs the agent may run on. Placement reserves cores + // for CPU replicas out of it, the way it reserves memory out of + // memory_total_mb; zero, from an older agent, reserves nothing. + int32 cpu_count = 18; // imex is the service that lets a multi-node NVLink domain share memory. IMEX imex = 15; @@ -420,6 +424,8 @@ message Heartbeat { // domain, and the node has to stop being offered work that spans it. repeated GPUFabricState gpu_fabric = 16; IMEX imex = 17; + // cpu_count as in Register, for a VM given more CPUs while it runs. + int32 cpu_count = 18; } // Provision is the software a node should have: which engines, at which @@ -530,6 +536,9 @@ message StartShard { // control plane from the replica id. The router sends it; nothing else // that can reach the node's ports has it. string api_key = 16; + // cpu_threads is how many CPU threads a shard with no GPU computes on: + // the cores placement reserved for it. Zero leaves it to the agent. + int32 cpu_threads = 17; } message EngineProfile { diff --git a/docs/reference.md b/docs/reference.md index bce4b1a..a1199f3 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -129,6 +129,7 @@ rather than repeated. | `args` | `[]string` | [Profile](#profile) | | `env` | `map[string]string` | [Profile](#profile) | | `min_memory_gb` | `int` | [Profile](#profile) | +| `cpu_cores` | `int` | [Profile](#profile) | | `min_disk_gb` | `int` | [Profile](#profile) | | `node_selector` | `map[string]string` | [Profile](#profile) | | `weight` | `float64` | [Profile](#profile) | @@ -966,6 +967,20 @@ this. A node that does not report its memory is not excluded either — an older agent must not have its nodes quietly emptied by an upgrade of the control plane. +### `cpu_cores` + +`int` + +CPUCores is the cores a replica of a profile with no GPU is given: +reserved out of its node's CPUs by placement, as memory is, and what +its engine computes on (llama.cpp's --threads). + +Reserved because an engine takes every core it is told about, and two +on one node that each think they have all of them spin against each +other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second +to under one each. Zero reserves nothing and leaves the thread count to +the node, which is every profile from before this. + ### `min_disk_gb` `int` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4801fa5..72afbf3 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "slices" "sync" "time" @@ -549,6 +550,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { // and a node that grew from 7 GB to 32 GB has to be seen to have grown // without anybody restarting anything. hb.MemoryTotalMb = int64(discovery.MemoryTotalMB()) + hb.CpuCount = int32(runtime.NumCPU()) a.reprobeEngines() a.mu.Lock() diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 4f7862a..52569bf 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -76,6 +76,10 @@ type ShardSpec struct { // and accounting. Given to the engine in its environment rather than on // its command line, which every user on the node can read. APIKey string `json:"api_key,omitempty"` + // Threads is the CPU cores placement reserved for a shard with no GPU, + // and so what its engine computes on. Zero leaves it to the engine's own + // configuration. + Threads int `json:"threads,omitempty"` // Pool and Fabric are control plane concepts the agent cannot derive. They // are carried so the agent can report them back after a restart; Fabric diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 7b500af..4de8060 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -133,7 +133,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { args = append(args, "--n-gpu-layers", "999") } if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") { - n := l.cfg.Threads + // What placement reserved first: it is what the node's other + // replicas were placed around. + n := s.Threads + if n <= 0 { + n = l.cfg.Threads + } if n <= 0 { n = max(numCPU()/2, 1) } diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index aff2d69..3cafd35 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -294,3 +294,15 @@ func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) { t.Errorf("--threads %q on one CPU", v) } } + +func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) { + s := llamaSpec(nil) + s.Threads = 3 + cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s) + if err != nil { + t.Fatal(err) + } + if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" { + t.Errorf("--threads %q; what placement reserved should win over the node's setting", v) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 45a6d3e..aadc972 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -139,6 +139,7 @@ func newCtlCmd() *cobra.Command { newStatusCmd(client, &asJSON), newModelCmd(client), newScaleCmd(client), + newResourcesCmd(client), newRestartCmd(client), newRetryCmd(client), newUpgradeCmd(client), @@ -323,17 +324,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS") + fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS") + var warnings []string for _, m := range models { serving := "no" if m.Ready > 0 { serving = "yes" } - fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n", - m.Name, serving, m.Ready, m.Min, modelStatus(m), + fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\n", + m.Name, serving, m.Ready, m.Min, modelStatus(m), reserves(m), join(m.Aliases), join(m.Profiles), join(m.Fallbacks)) + for _, x := range m.Warnings { + warnings = append(warnings, m.Name+": "+x) + } } - return w.Flush() + if err := w.Flush(); err != nil { + return err + } + for _, x := range warnings { + fmt.Println("warning:", x) + } + return nil }, } } @@ -511,3 +522,22 @@ func dash(s string) string { } return s } + +// reserves is what one replica of a model holds of a node, with what it was +// measured using beside it: "6G 2c (5.1G used)". +func reserves(m adminapi.ModelView) string { + if m.MemoryGB == 0 && m.CPUCores == 0 { + return "-" + } + out := fmt.Sprintf("%dG", m.MemoryGB) + if m.CPUCores > 0 { + out += fmt.Sprintf(" %dc", m.CPUCores) + } + if m.MeasuredMemoryGB > 0 { + out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB) + } + if len(m.Warnings) > 0 { + out += " !" + } + return out +} diff --git a/internal/cli/model.go b/internal/cli/model.go index e0346a3..fad3dbb 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -192,3 +192,44 @@ func newScaleCmd(client func() *ctlClient) *cobra.Command { "how long an on-demand model stays up unasked-for; zero takes the cluster default") return c } + +func newResourcesCmd(client func() *ctlClient) *cobra.Command { + var memory, cores int + var profile string + c := &cobra.Command{ + Use: "resources --memory GB [--cores N]", + Short: "Set what a model reserves of a node: memory, and CPU cores", + Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" + + "weights, KV cache and engine together. Placement reserves it on the node,\n" + + "so a node holds only as many replicas as it has memory for.\n\n" + + "--cores is the CPU cores a replica with no GPU computes on. They are\n" + + "reserved the same way, and become the engine's thread count, so two\n" + + "replicas on one node stop competing for the same cores. Zero reserves\n" + + "none and leaves the thread count to the node.\n\n" + + "Running replicas keep what they were started with; `llmgw ctl restart\n" + + "--model M` moves them to the new figures.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("memory") { + return fmt.Errorf("--memory is required") + } + body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores}) + if err != nil { + return err + } + var m catalog.Model + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil { + return err + } + for _, p := range m.Profiles { + fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores) + } + fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name) + return nil + }, + } + c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB") + c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none") + c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out") + return c +} diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index f5a452c..174913b 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -75,6 +75,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run") f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for") f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots") + f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits") f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use") return c } diff --git a/internal/control/adminapi/actor_test.go b/internal/control/adminapi/actor_test.go index 1452ce6..cbc355b 100644 --- a/internal/control/adminapi/actor_test.go +++ b/internal/control/adminapi/actor_test.go @@ -4,6 +4,8 @@ import ( "net/http/httptest" "strings" "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) func TestAClaimedActorIsOneShortName(t *testing.T) { @@ -22,3 +24,22 @@ func TestAClaimedActorIsOneShortName(t *testing.T) { } } } + +func TestAReservationThatPlacementCannotTrustIsSaid(t *testing.T) { + cpu := func(mem int) catalog.Model { + return catalog.Model{Name: "m", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: mem}}} + } + if w := reservationWarnings(cpu(0), 0); len(w) != 1 || !strings.Contains(w[0], "declares no memory") { + t.Errorf("no memory declared: %v", w) + } + if w := reservationWarnings(cpu(6), 8.2); len(w) != 1 || !strings.Contains(w[0], "uses 8.2 GB and reserves 6") { + t.Errorf("under-declared: %v", w) + } + if w := reservationWarnings(cpu(6), 6.4); len(w) != 0 { + t.Errorf("within a tenth is mapped weights, not a wrong figure: %v", w) + } + gpu := catalog.Model{Name: "g", Profiles: []catalog.Profile{{Name: "tp1", Engine: "vllm", GPUClass: "h100-80g"}}} + if w := reservationWarnings(gpu, 3); len(w) != 0 { + t.Errorf("a GPU model needs no host memory figure: %v", w) + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index cc817ff..ccc1ab5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -57,6 +57,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models", a.addModel) mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel) mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) + mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources) mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity) mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) mux.HandleFunc("GET /api/v1/usage", a.usageByScope) @@ -119,6 +120,8 @@ type NodeView struct { // what decides whether a CPU profile can be placed here, so it belongs // where someone looks when a model will not schedule. MemoryGB int `json:"memory_gb,omitempty"` + // CPUCount is the machine's CPUs, 0 when the agent did not report them. + CPUCount int `json:"cpu_count,omitempty"` // Engines are the inference engines installed here, as the agent found // them. Absent means the agent is older than the probe and said nothing, // which is not the same as having none — placement treats it that way too, @@ -270,6 +273,15 @@ type ModelView struct { Unplaceable bool `json:"unplaceable,omitempty"` Fallbacks []string `json:"fallbacks,omitempty"` Profiles []string `json:"profiles"` + // MemoryGB and CPUCores are what one replica reserves, from its first + // profile; MeasuredMemoryGB is the most one of its replicas uses now, 0 + // when none is measured. + MemoryGB int `json:"memory_gb,omitempty"` + CPUCores int `json:"cpu_cores,omitempty"` + MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"` + // Warnings are what makes its reservation wrong: nothing declared where + // placement needs a figure, or less declared than it uses. + Warnings []string `json:"warnings,omitempty"` } // ReplicaView is a replica as `ctl replicas` shows it. @@ -396,8 +408,8 @@ func (a *API) Nodes() []NodeView { ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr, State: n.State.String(), Labels: n.Labels, GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)), - MemoryGB: n.MemoryTotalMB / 1024, - Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features, + MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount, + Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features, Replicas: replicasOn[n.ID], } if free, total := n.LocalDisk(); total > 0 { @@ -497,6 +509,15 @@ func (a *API) Models() []ModelView { for _, h := range a.ctl.Reconciler().ModelHealth() { health[h.Model] = h } + // The most one replica of each model uses, summed over its shards. + measured := map[string]int64{} + for _, rep := range a.Replicas("") { + var b int64 + for _, sh := range rep.Shards { + b += sh.MemoryBytes + } + measured[rep.Model] = max(measured[rep.Model], b) + } out := make([]ModelView, 0) for _, m := range a.ctl.Catalog().Models() { h := health[m.Name] @@ -511,6 +532,11 @@ func (a *API) Models() []ModelView { for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) } + if len(m.Profiles) > 0 { + v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores + } + v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30) + v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB) out = append(out, v) } return out @@ -581,6 +607,15 @@ type Models interface { Add(ctx context.Context, m catalog.Model, by string) error Remove(ctx context.Context, name string) error Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) + Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) +} + +// ResourcesRequest sets what a model's profiles reserve of a node. +type ResourcesRequest struct { + // Profile is one profile's name; empty is every profile. + Profile string `json:"profile,omitempty"` + MemoryGB int `json:"memory_gb"` + CPUCores int `json:"cpu_cores"` } // Usage reports what each group of the tenancy tree spent. The store @@ -826,6 +861,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) { }) } +// setResources sets a model's reserved memory and cores. Placed replicas keep +// what they were started with until they are restarted. +func (a *API) setResources(w http.ResponseWriter, r *http.Request) { + if a.models == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + var req ResourcesRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, m) +} + func (a *API) removeModel(w http.ResponseWriter, r *http.Request) { if a.models == nil { writeError(w, http.StatusNotImplemented, "this control plane has no store") @@ -1307,3 +1362,40 @@ func actorOf(r *http.Request) string { } return b.String() } + +// NodeCPUs is the CPU counts of ready nodes with at least this much memory: +// where a replica needing it could go. +func (a *API) NodeCPUs(minMemoryGB int) []int { + var out []int + for _, n := range a.ctl.Inventory().Nodes() { + if n.State.String() != "ready" || n.CPUCount == 0 { + continue + } + if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB { + continue + } + out = append(out, n.CPUCount) + } + return out +} + +// reservationWarnings says what is wrong with what a model reserves. +// +// Resident memory includes the weights mapped from disk, which the kernel can +// take back, so it overstates what a model needs by the part of the weights +// not in use. A tenth over is let pass for that; more is a figure too low for +// placement to be trusted with. +func reservationWarnings(m catalog.Model, measuredGB float64) []string { + var out []string + for _, p := range m.Profiles { + if p.CPUOnly() && p.MinMemoryGB <= 0 { + out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+ + "(llmgw ctl model resources %s --memory GB)", p.Name, m.Name)) + } + } + if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 { + out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory", + measuredGB, m.Profiles[0].MinMemoryGB)) + } + return out +} diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 21cc382..b8dea81 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -258,6 +258,13 @@ type fakeModels struct { err error } +func (f *fakeModels) Resources(_ context.Context, name, profile string, mem, cores int, _ string) (catalog.Model, error) { + if f.err != nil { + return catalog.Model{}, f.err + } + return catalog.Model{Name: name, Profiles: []catalog.Profile{{Name: "p", MinMemoryGB: mem, CPUCores: cores}}}, nil +} + func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) { if f.err != nil { return catalog.Model{}, f.err diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 8bfba45..9af2a31 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -4,6 +4,7 @@ package agentsvc import ( "fmt" "maps" + "runtime" "time" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" @@ -27,6 +28,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { EngineVersions: r.GetEngineVersions(), EnginesProbed: r.GetEnginesProbed(), MemoryTotalMB: int(r.GetMemoryTotalMb()), + CPUCount: int(r.GetCpuCount()), } if h := r.GetHost(); h != nil { n.Host = discovery.Host{ @@ -90,6 +92,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr, Labels: labels, AgentVersion: agentVersion, MemoryTotalMb: int64(inv.MemoryTotalMB), + CpuCount: int32(runtime.NumCPU()), } for _, w := range inv.Weights { r.Weights = append(r.Weights, &llmgwv1.LocalWeights{ @@ -192,6 +195,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { Fabric: cmd.Fabric, ProfileFp: cmd.ProfileFP, ApiKey: cmd.Spec.APIKey, + CpuThreads: int32(cmd.Spec.Threads), } } @@ -213,6 +217,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration Fabric: m.GetFabric(), ProfileFP: m.GetProfileFp(), APIKey: m.GetApiKey(), + Threads: int(m.GetCpuThreads()), Profile: engines.Profile{ Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(), TP: int(p.GetTp()), PP: int(p.GetPp()), diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 2655532..68de0e8 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -345,8 +345,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { problems := hb.GetNewProblems() storage := hb.GetStorage() memMB := int(hb.GetMemoryTotalMb()) + cpus := int(hb.GetCpuCount()) if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && - !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() && + !hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() && !hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 && !hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil { return @@ -406,6 +407,12 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { "from_mb", n.MemoryTotalMB, "to_mb", memMB) n.MemoryTotalMB = memMB } + if cpus > 0 && cpus != n.CPUCount { + if n.CPUCount > 0 { + s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus) + } + n.CPUCount = cpus + } if hb.GetEnginesChanged() { vers := maps.Clone(hb.GetEngineVersions()) if !maps.Equal(vers, n.EngineVersions) { diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index b64277c..95ae8b0 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -74,6 +74,17 @@ type Profile struct { // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` + // CPUCores is the cores a replica of a profile with no GPU is given: + // reserved out of its node's CPUs by placement, as memory is, and what + // its engine computes on (llama.cpp's --threads). + // + // Reserved because an engine takes every core it is told about, and two + // on one node that each think they have all of them spin against each + // other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second + // to under one each. Zero reserves nothing and leaves the thread count to + // the node, which is every profile from before this. + CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"` + // MinDiskGB is the free space a node needs before it can hold these // weights. // @@ -380,6 +391,7 @@ func (m Model) PlacementProfiles() []placement.Profile { NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, MinDiskGB: p.MinDiskGB, + CPUCores: p.CPUCores, // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, @@ -475,3 +487,19 @@ func (p Profile) Fingerprint() string { sum := sha256.Sum256(b) return hex.EncodeToString(sum[:8]) } + +// SuggestCores is the cores to give a replica with no GPU on nodes of these +// CPU counts: half of the smallest, so two replicas share any of them without +// competing. Zero when no node says, which reserves nothing. +func SuggestCores(cpuCounts []int) int { + smallest := 0 + for _, c := range cpuCounts { + if c > 0 && (smallest == 0 || c < smallest) { + smallest = c + } + } + if smallest == 0 { + return 0 + } + return max(smallest/2, 1) +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 8bfcce5..dc92159 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -79,6 +79,9 @@ type Node struct { // it, which is treated as no constraint — the same as every node before // this field existed. MemoryTotalMB int `json:"memory_total_mb,omitempty"` + // CPUCount is the CPUs the agent may run on. Zero is an agent too old to + // say, which, like memory, constrains nothing. + CPUCount int `json:"cpu_count,omitempty"` // Weights are the model files this node can see, however they got there. // Empty means the node was not asked to look, which is not the same as @@ -373,6 +376,9 @@ type Allocation struct { // Zero for a profile that declared nothing, which constrains nothing and // is counted as nothing. MemoryGB int + // CPUCores is the cores this shard was given to compute on, for a shard + // with no GPU; zero for one that shares whatever is free. + CPUCores int } // Store holds the cluster inventory. It is safe for concurrent use. diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 68429e9..6d22a2d 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -91,6 +91,9 @@ type Request struct { // use. They default to 16384 and 4096, what four parallel slots divide. CtxPool int `json:"ctx_pool,omitempty"` ContextLength int `json:"context_length,omitempty"` + // CPUCores is what a replica computes on; zero takes half the smallest + // node it fits on. + CPUCores int `json:"cpu_cores,omitempty"` // By is who asked, for the catalog's record. By string `json:"-"` } @@ -278,13 +281,20 @@ func (s *Server) run(j *Job) { } s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis)) + cores := req.CPUCores + if cores <= 0 { + if nc, ok := s.deps.Fleet.(interface{ NodeCPUs(int) []int }); ok { + cores = catalog.SuggestCores(nc.NodeCPUs(memGB)) + } + } + // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, Profiles: []catalog.Profile{{ Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, - Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, + Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, }}, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, } diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 94c7621..dbdbf61 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -18,6 +18,7 @@ package modelsvc import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "sync" @@ -104,6 +105,16 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error { if s.Declared(m.Name) { return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name) } + // A model with no GPU is all host memory, and one that declares none + // fits everywhere as far as placement can tell — however many of it + // there already are. Required of what is added now; models stored before + // this still load, and say what they are missing. + for _, p := range m.Profiles { + if p.CPUOnly() && p.MinMemoryGB <= 0 { + return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+ + "placement could not tell when a node is full", m.Name, p.Name) + } + } if s.check != nil { if err := s.check(ctx, m); err != nil { return err @@ -189,36 +200,72 @@ func (s *Service) Remove(ctx context.Context, name string) error { return nil } -// Scale changes how many replicas a model runs, without a restart. -// -// Only the scaling block moves. Everything else about the model — its source, -// its profiles, the engine arguments somebody tuned — is read back from what -// was stored and written out again untouched, so changing a number cannot -// quietly become a redefinition. -// -// A model declared in configuration is refused, for the same reason Remove -// refuses one: the YAML is read on every start, so the new number would hold -// until the next restart and then silently revert. A cluster that disagrees -// with its own config file is worse than an error message saying where to edit. +// Scale changes how many replicas a model added at runtime runs. func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error { + m.Scaling = sc + return nil + }) + if err == nil { + s.log.Info("model scaled", "model", name, + "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by) + } + return final, err +} + +// Resources sets what a model's profile reserves of a node: host memory, and +// the CPU cores a replica without a GPU computes on. An empty profile name +// sets every profile. +func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) { + if memoryGB < 0 || cpuCores < 0 { + return catalog.Model{}, errors.New("memory and cores cannot be negative") + } + final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error { + found := false + for i := range m.Profiles { + if profile != "" && m.Profiles[i].Name != profile { + continue + } + found = true + m.Profiles[i].MinMemoryGB = memoryGB + m.Profiles[i].CPUCores = cpuCores + } + if !found { + return fmt.Errorf("model %q has no profile %q", name, profile) + } + return nil + }) + if err == nil { + s.log.Info("model resources set", "model", name, "profile", profile, + "memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by) + } + return final, err +} + +// update changes one model added at runtime and keeps the change. +// +// Built before anything is written, like Add: catalog.New is what applies the +// defaults and refuses what does not validate, so the model that comes back +// is the one that will run rather than the one that was asked for. +func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) { s.mu.Lock() defer s.mu.Unlock() if s.Declared(name) { return catalog.Model{}, fmt.Errorf( - "model %q is declared in configuration; change its scaling there, or it reverts on the next restart", name) + "model %q is declared in configuration; %s there, or it reverts on the next restart", name, what) } - stored, err := s.storedModels(ctx) if err != nil { return catalog.Model{}, err } - next := append([]catalog.Model{}, s.declared...) found := false for _, e := range stored { if e.Name == name { - e.Scaling = sc + if err := change(&e); err != nil { + return catalog.Model{}, err + } found = true } next = append(next, e) @@ -226,20 +273,14 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by if !found { return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name) } - - // Built before anything is written, like Add: catalog.New is what applies - // the defaults (max follows min when it is left out) and what refuses a - // max below a min, so the model that comes back is the one that will run - // rather than the one that was asked for. built, err := catalog.New(next) if err != nil { - return catalog.Model{}, fmt.Errorf("scaling %q that way would make the catalog invalid: %w", name, err) + return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err) } final, ok := built.Resolve(name) if !ok { return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name) } - def, err := json.Marshal(final) if err != nil { return catalog.Model{}, err @@ -249,10 +290,7 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by }); err != nil { return catalog.Model{}, err } - s.apply.ReplaceCatalog(ctx, built) - s.log.Info("model scaled", "model", name, - "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by) return final, nil } diff --git a/internal/control/modelsvc/modelsvc_test.go b/internal/control/modelsvc/modelsvc_test.go index 63b49dd..bd72e95 100644 --- a/internal/control/modelsvc/modelsvc_test.go +++ b/internal/control/modelsvc/modelsvc_test.go @@ -48,7 +48,7 @@ func model(name string) catalog.Model { return catalog.Model{ Name: name, Source: "/models/" + name, Scaling: catalog.Scaling{Min: 1, Max: 1}, - Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}}, + Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1, MinMemoryGB: 2}}, } } @@ -337,3 +337,43 @@ func TestMinZeroWithAMaxIsOnDemand(t *testing.T) { t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout) } } + +func TestACPUModelAddedNowMustSayHowMuchMemoryItNeeds(t *testing.T) { + s, st, _ := newService(t) + m := model("bare") + m.Profiles[0].MinMemoryGB = 0 + err := s.Add(t.Context(), m, "tester") + if err == nil || !strings.Contains(err.Error(), "declares no memory") { + t.Fatalf("a CPU model with no memory was added: %v", err) + } + if len(st.rows) != 0 { + t.Error("it was stored anyway") + } +} + +func TestResourcesAreSetOnAStoredModelAndSurviveARestart(t *testing.T) { + s, _, _ := newService(t) + if err := s.Add(t.Context(), model("m"), "tester"); err != nil { + t.Fatal(err) + } + got, err := s.Resources(t.Context(), "m", "", 6, 3, "tester") + if err != nil { + t.Fatal(err) + } + if p := got.Profiles[0]; p.MinMemoryGB != 6 || p.CPUCores != 3 { + t.Errorf("profile %+v", p) + } + if _, err := s.Resources(t.Context(), "m", "nope", 6, 3, "tester"); err == nil { + t.Error("a profile the model does not have was accepted") + } + if _, err := s.Resources(t.Context(), "m", "", -1, 0, "tester"); err == nil { + t.Error("negative memory was accepted") + } + cat, err := s.Load(t.Context()) + if err != nil { + t.Fatal(err) + } + if m, _ := cat.Resolve("m"); m.Profiles[0].CPUCores != 3 { + t.Error("the cores were lost on reload") + } +} diff --git a/internal/control/placement/cpu_test.go b/internal/control/placement/cpu_test.go index d5c4891..5f72dc1 100644 --- a/internal/control/placement/cpu_test.go +++ b/internal/control/placement/cpu_test.go @@ -1,6 +1,7 @@ package placement import ( + "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" @@ -100,3 +101,38 @@ func TestCPUProfileRespectsAvoid(t *testing.T) { t.Errorf("placed on %q despite being told to avoid it", got) } } + +func TestCPUCoresAreReservedAsMemoryIs(t *testing.T) { + n := node("n1", "", 0) + n.CPUCount = 8 + st := store(n) + s := New(st) + p := cpuProfile("cpu") + p.CPUCores = 3 + + for i := range 2 { + res := s.Place([]Profile{p}, nil) + if res.Placement == nil { + t.Fatalf("replica %d did not fit in 8 cores:\n%s", i+1, res.Explain()) + } + st.Allocate("n1", inventory.Allocation{ShardID: string(rune('a' + i)), CPUCores: 3}) + } + // Six of eight given away: a third would need three. + res := s.Place([]Profile{p}, nil) + if res.Placement != nil { + t.Fatal("a third replica was given cores the node does not have") + } + if got := res.Explain(); !strings.Contains(got, "has 8 CPUs with 2 free, needs 3") { + t.Errorf("the refusal should say how many cores are left:\n%s", got) + } + + // Unknown counts constrain nothing: an older agent, or a profile that + // reserves no cores. + old := node("n2", "", 0) + if res := New(store(old)).Place([]Profile{p}, nil); res.Placement == nil { + t.Error("a node that does not report its CPUs was excluded") + } + if res := s.Place([]Profile{cpuProfile("cpu")}, nil); res.Placement == nil { + t.Error("a profile reserving no cores was refused") + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 59ee579..fdb92d7 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -39,6 +39,9 @@ type Profile struct { // MinDiskGB is the free space needed to hold the weights, checked only on // nodes that do not already have them. Zero is unconstrained. MinDiskGB int + // CPUCores is the cores a profile with no GPU reserves. Zero is + // unconstrained. + CPUCores int // Source is the model's weights, as the engine would be given it. Used to // prefer nodes that already hold them; empty asks for no preference. Source string @@ -582,6 +585,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje } } + // Cores, for a replica that computes on them: what is left after the + // cores already given to the node's other replicas. Same exemptions + // as memory: zero declares nothing, and a node that does not report + // its CPUs is an older agent, not one with none. + if p.CPUOnly() && p.CPUCores > 0 && n.CPUCount > 0 { + free := n.CPUCount - s.claimedCores(n.ID) + if free < p.CPUCores { + reject(fmt.Sprintf("has %d CPUs with %d free, needs %d", n.CPUCount, free, p.CPUCores)) + continue + } + } + if p.CPUOnly() { // Every ready node can run it; there is no card to be free. out = append(out, candidate{node: n}) @@ -723,6 +738,15 @@ func (s *Scheduler) claimedMemory(nodeID string) int { return total } +// claimedCores is the CPU cores already given to replicas on a node. +func (s *Scheduler) claimedCores(nodeID string) int { + total := 0 + for _, a := range s.inv.Allocations(nodeID) { + total += a.CPUCores + } + return total +} + // installing says why an engine is missing, when the node has said. func installing(n inventory.Node, engine string) string { for _, p := range n.Provision { diff --git a/internal/control/reconciler/adopt.go b/internal/control/reconciler/adopt.go index 8a27c38..b5830e3 100644 --- a/internal/control/reconciler/adopt.go +++ b/internal/control/reconciler/adopt.go @@ -144,6 +144,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) { r.inv.Allocate(s.NodeID, inventory.Allocation{ ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices, MemoryGB: r.profileMemory(rep.Model, rep.Profile), + CPUCores: r.adoptedCores(rep.Model, rep.Profile), }) // The endpoint comes from the leader, which may be in any of the // groups — the rank 0 node is not necessarily the first to register. @@ -253,3 +254,9 @@ type AdoptedShard struct { // Endpoint is the leader's base URL, set on the rank 0 shard. Endpoint string } + +// adoptedCores is what an adopted shard is taken to hold of its node's CPUs. +func (r *Reconciler) adoptedCores(model, profile string) int { + p, _ := r.profileOf(model, profile) + return cpuCores(p) +} diff --git a/internal/control/reconciler/headroom.go b/internal/control/reconciler/headroom.go index fae46a5..6080b03 100644 --- a/internal/control/reconciler/headroom.go +++ b/internal/control/reconciler/headroom.go @@ -66,14 +66,14 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { h.Blocked = slices.Compact(h.Blocked) return h, true } - mem := 0 + mem, cores := 0, 0 if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok { - mem = prof.MinMemoryGB + mem, cores = prof.MinMemoryGB, cpuCores(prof) } var nodes []string for _, a := range res.Placement.Nodes { inv.Allocate(a.NodeID, inventory.Allocation{ - ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, + ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, }) nodes = append(nodes, a.NodeID) } diff --git a/internal/control/reconciler/headroom_test.go b/internal/control/reconciler/headroom_test.go index 0bf3c1d..44f9111 100644 --- a/internal/control/reconciler/headroom_test.go +++ b/internal/control/reconciler/headroom_test.go @@ -57,3 +57,24 @@ func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) { t.Errorf("headroom %+v, expected room for 3 more", got) } } + +func TestReservedCoresAreTheEnginesThreadsAndBoundTheRoom(t *testing.T) { + cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 64 * 1024, CPUCount: 8} + m := catalog.Model{Name: "coder", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6, CPUCores: 3}}} + h := newHarness(t, m, cpu) + h.r.Reconcile(t.Context()) + + started := h.disp.startedShards() + if len(started) != 1 || started[0].Spec.Threads != 3 { + t.Fatalf("the engine was not told its 3 cores: %+v", started) + } + if a := h.inv.Allocations("a"); len(a) != 1 || a[0].CPUCores != 3 { + t.Errorf("the cores were not reserved: %+v", a) + } + // Memory would allow nine more; cores allow one: 8 - 3 = 5, then 2. + got, _ := h.r.Headroom("coder", 10) + if got.More != 1 { + t.Errorf("room for %d more, expected 1 by cores: %+v", got.More, got) + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 9f86bd6..8572f08 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -825,7 +825,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac }) r.inv.Allocate(a.NodeID, inventory.Allocation{ ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices, - MemoryGB: prof.MinMemoryGB, + MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), }) } rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort) @@ -869,6 +869,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac HTTPPort: s.HTTPPort, Profile: espec, APIKey: r.engineKey(id), + Threads: cpuCores(prof), }, StartupTimeout: prof.StartupTimeout, Pool: pl.Pool, @@ -1108,16 +1109,31 @@ func addrOf(n inventory.Node) string { // claim on restart would place a second large model onto a node already full of // the first. func (r *Reconciler) profileMemory(model, profile string) int { + p, _ := r.profileOf(model, profile) + return p.MinMemoryGB +} + +// profileOf is a replica's profile as the catalog holds it now. +func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) { m, ok := r.cat.Load().Resolve(model) if !ok { - return 0 + return catalog.Profile{}, false } for _, p := range m.Profiles { if p.Name == profile { - return p.MinMemoryGB + return p, true } } - return 0 + return catalog.Profile{}, false +} + +// cpuCores is what a profile reserves of a node's CPUs: its cores when it +// computes on them, nothing when a GPU does the work. +func cpuCores(p catalog.Profile) int { + if !p.CPUOnly() { + return 0 + } + return p.CPUCores } func (r *Reconciler) engineKey(replicaID string) string { diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 98cfa1e..4ac7666 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -49,6 +49,9 @@ type serveForm struct { // 3.5 GB for a KV pool. Nothing was wrong with placement — it was never // told what the model needed. MinMemoryGB int + // CPUCores is the cores a replica computes on: half the smallest node it + // fits on, or zero when no node says how many it has. + CPUCores int // MemoryBasis says how MinMemoryGB was arrived at, for the form to show. MemoryBasis string // CtxPool is the engine context pool the form starts with, and the one @@ -152,6 +155,15 @@ func planServe(st mirror.Status, files []mirror.Status, base string, defaultCtxPool, header.BlockCount, kvHeads, float64(int64(1<<30)+weights/20)/(1<<30)) } + if fleet != nil { + var cpus []int + for _, n := range fleet.Nodes() { + if n.State == "ready" && (n.MemoryGB == 0 || n.MemoryGB >= f.MinMemoryGB) { + cpus = append(cpus, n.CPUCount) + } + } + f.CPUCores = catalog.SuggestCores(cpus) + } if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest { f.Warning = strings.TrimSuffix(f.Warning, ".") + fmt.Sprintf("%sthe largest node has %d GB and this needs about %d", @@ -277,6 +289,11 @@ func buildModel(get func(string) string) (catalog.Model, error) { return m, err } } + if v := strings.TrimSpace(get("cpu_cores")); v != "" { + if p.CPUCores, err = positive(v, "the cores"); err != nil { + return m, err + } + } if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" { // What keeps a large model off the small nodes. A CPU profile has no // card to be short of, so without this every ready node looks like a diff --git a/internal/web/static/base.css b/internal/web/static/base.css index b98f799..489bc2e 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -854,3 +854,4 @@ table.explore td:first-child { max-width: 34rem; overflow: hidden; text-overflow table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted); } .clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis; vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; } +table.rows td.warn { color: var(--warn); cursor: help; } diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index a97349c..42aba77 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -165,7 +165,9 @@ reachable from this machine.

+ + diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 116621d..026a9a6 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -80,12 +80,14 @@

Models

{{if .Models}} -{{if .AnyFallbacks}}{{end}} +
ModelAliasesReadyWantedProfilesFallbacksWhat it is for
{{if .AnyFallbacks}}{{end}} {{range .Models}} + {{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}} + {{- if .MeasuredMemoryGB}} {{printf "%.1f" .MeasuredMemoryGB}}G used{{end}}{{if .Warnings}} ⚠{{end}} {{if $.AnyFallbacks}}{{end}} {{/* Cut to the column, the whole sentence on hover: a row of its own for
ModelAliasesReadyWantedReservesProfilesFallbacksWhat it is for
{{.Name}} {{join .Aliases}} {{.Ready}} {{.Min}}–{{.Max}}{{join .Profiles}}{{join .Fallbacks}}