diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddf293..3f593e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,30 @@ APIs change between minors without a deprecation period. Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until there are enough for v0.10.5. +### CPU cores are reserved, and memory has to be declared + +The first step of resource allocation for larger clusters: what a replica +holds of a node is reserved, not assumed. + +- **Cores.** A profile's `cpu_cores` is reserved out of its node's CPUs, as + memory is, and is what its engine computes on (llama.cpp's `--threads`). A + node holds as many CPU replicas as it has cores for, and two on one node no + longer spin against each other. Agents report their CPU count; a node that + does not, and a profile that declares no cores, constrain nothing. +- **Memory is required of new CPU models.** A model with no GPU is all host + memory, and one declaring none looked to placement as if it fitted + everywhere, any number of times. Adding one without `min_memory_gb` is now + refused. Models stored before this still load, and say what is missing. +- **Declared against measured.** Each model shows what one replica reserves + and the most one of its replicas uses — in `ctl models` (RESERVES) and on the + cluster page — with a warning when nothing is declared or when it uses more + than a tenth over what it reserves. +- **Setting them.** `llmgw ctl model resources MODEL --memory GB --cores N` + (and `POST /api/v1/models/{name}/resources`) for a model added at runtime. + `ctl model serve` and the mirror's serve form fill in cores — half the + smallest node the model fits on — beside the memory they already estimate. + Running replicas keep their old figures until restarted. + ### The mirror page in tabs /admin/mirror was one long page: what the mirror holds, the Hugging Face diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index b0ef34a..41adf67 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -534,6 +534,10 @@ type Register struct { // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"` + // cpu_count is the CPUs the agent may run on. Placement reserves cores + // for CPU replicas out of it, the way it reserves memory out of + // memory_total_mb; zero, from an older agent, reserves nothing. + CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"` // imex is the service that lets a multi-node NVLink domain share memory. Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"` // installable_engines are the engines this agent knows how to install @@ -679,6 +683,13 @@ func (x *Register) GetEnginesProbed() bool { return false } +func (x *Register) GetCpuCount() int32 { + if x != nil { + return x.CpuCount + } + return 0 +} + func (x *Register) GetImex() *IMEX { if x != nil { return x.Imex @@ -2017,8 +2028,10 @@ type Heartbeat struct { // gpu_fabric is each GPU's NVLink domain membership as it stands now, sent // once a minute: a fabric manager restarting drops every card out of its // domain, and the node has to stop being offered work that spans it. - GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"` - Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"` + GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"` + Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"` + // cpu_count as in Register, for a VM given more CPUs while it runs. + CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2172,6 +2185,13 @@ func (x *Heartbeat) GetImex() *IMEX { return nil } +func (x *Heartbeat) GetCpuCount() int32 { + if x != nil { + return x.CpuCount + } + return 0 +} + // Provision is the software a node should have: which engines, at which // versions. Nothing more. // @@ -2561,7 +2581,10 @@ type StartShard struct { // api_key is what the engine demands of every request, derived by the // control plane from the replica id. The router sends it; nothing else // that can reach the node's ports has it. - ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"` + ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"` + // cpu_threads is how many CPU threads a shard with no GPU computes on: + // the cores placement reserved for it. Zero leaves it to the agent. + CpuThreads int32 `protobuf:"varint,17,opt,name=cpu_threads,json=cpuThreads,proto3" json:"cpu_threads,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2708,6 +2731,13 @@ func (x *StartShard) GetApiKey() string { return "" } +func (x *StartShard) GetCpuThreads() int32 { + if x != nil { + return x.CpuThreads + } + return 0 +} + type EngineProfile struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -3655,7 +3685,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "fabricTest\x122\n" + "\aupgrade\x18\n" + " \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" + - "\x03msg\"\xe3\x06\n" + + "\x03msg\"\x80\a\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -3671,7 +3701,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" + "\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" + - "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" + + "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\x1b\n" + + "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x12\"\n" + "\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" + "\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x12\x1a\n" + "\bfeatures\x18\x11 \x03(\tR\bfeatures\x1a9\n" + @@ -3811,7 +3842,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06status\x18\x03 \x01(\tR\x06status\x12!\n" + "\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" + "\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" + - "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd9\x06\n" + + "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xf6\x06\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -3831,7 +3862,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x127\n" + "\n" + "gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" + - "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1aA\n" + + "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" + + "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + @@ -3872,7 +3904,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + "\vbytes_total\x18\x03 \x01(\x03R\n" + - "bytesTotal\"\xf2\x03\n" + + "bytesTotal\"\x93\x04\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -3896,7 +3928,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" + "\n" + "profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" + - "\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" + + "\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" + + "\vcpu_threads\x18\x11 \x01(\x05R\n" + + "cpuThreads\"\xfd\x02\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 811f095..cd24d04 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -120,6 +120,10 @@ message Register { // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. bool engines_probed = 14; + // cpu_count is the CPUs the agent may run on. Placement reserves cores + // for CPU replicas out of it, the way it reserves memory out of + // memory_total_mb; zero, from an older agent, reserves nothing. + int32 cpu_count = 18; // imex is the service that lets a multi-node NVLink domain share memory. IMEX imex = 15; @@ -420,6 +424,8 @@ message Heartbeat { // domain, and the node has to stop being offered work that spans it. repeated GPUFabricState gpu_fabric = 16; IMEX imex = 17; + // cpu_count as in Register, for a VM given more CPUs while it runs. + int32 cpu_count = 18; } // Provision is the software a node should have: which engines, at which @@ -530,6 +536,9 @@ message StartShard { // control plane from the replica id. The router sends it; nothing else // that can reach the node's ports has it. string api_key = 16; + // cpu_threads is how many CPU threads a shard with no GPU computes on: + // the cores placement reserved for it. Zero leaves it to the agent. + int32 cpu_threads = 17; } message EngineProfile { diff --git a/docs/reference.md b/docs/reference.md index bce4b1a..a1199f3 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -129,6 +129,7 @@ rather than repeated. | `args` | `[]string` | [Profile](#profile) | | `env` | `map[string]string` | [Profile](#profile) | | `min_memory_gb` | `int` | [Profile](#profile) | +| `cpu_cores` | `int` | [Profile](#profile) | | `min_disk_gb` | `int` | [Profile](#profile) | | `node_selector` | `map[string]string` | [Profile](#profile) | | `weight` | `float64` | [Profile](#profile) | @@ -966,6 +967,20 @@ this. A node that does not report its memory is not excluded either — an older agent must not have its nodes quietly emptied by an upgrade of the control plane. +### `cpu_cores` + +`int` + +CPUCores is the cores a replica of a profile with no GPU is given: +reserved out of its node's CPUs by placement, as memory is, and what +its engine computes on (llama.cpp's --threads). + +Reserved because an engine takes every core it is told about, and two +on one node that each think they have all of them spin against each +other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second +to under one each. Zero reserves nothing and leaves the thread count to +the node, which is every profile from before this. + ### `min_disk_gb` `int` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4801fa5..72afbf3 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "slices" "sync" "time" @@ -549,6 +550,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { // and a node that grew from 7 GB to 32 GB has to be seen to have grown // without anybody restarting anything. hb.MemoryTotalMb = int64(discovery.MemoryTotalMB()) + hb.CpuCount = int32(runtime.NumCPU()) a.reprobeEngines() a.mu.Lock() diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 4f7862a..52569bf 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -76,6 +76,10 @@ type ShardSpec struct { // and accounting. Given to the engine in its environment rather than on // its command line, which every user on the node can read. APIKey string `json:"api_key,omitempty"` + // Threads is the CPU cores placement reserved for a shard with no GPU, + // and so what its engine computes on. Zero leaves it to the engine's own + // configuration. + Threads int `json:"threads,omitempty"` // Pool and Fabric are control plane concepts the agent cannot derive. They // are carried so the agent can report them back after a restart; Fabric diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 7b500af..4de8060 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -133,7 +133,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { args = append(args, "--n-gpu-layers", "999") } if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") { - n := l.cfg.Threads + // What placement reserved first: it is what the node's other + // replicas were placed around. + n := s.Threads + if n <= 0 { + n = l.cfg.Threads + } if n <= 0 { n = max(numCPU()/2, 1) } diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index aff2d69..3cafd35 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -294,3 +294,15 @@ func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) { t.Errorf("--threads %q on one CPU", v) } } + +func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) { + s := llamaSpec(nil) + s.Threads = 3 + cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s) + if err != nil { + t.Fatal(err) + } + if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" { + t.Errorf("--threads %q; what placement reserved should win over the node's setting", v) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 45a6d3e..aadc972 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -139,6 +139,7 @@ func newCtlCmd() *cobra.Command { newStatusCmd(client, &asJSON), newModelCmd(client), newScaleCmd(client), + newResourcesCmd(client), newRestartCmd(client), newRetryCmd(client), newUpgradeCmd(client), @@ -323,17 +324,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS") + fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS") + var warnings []string for _, m := range models { serving := "no" if m.Ready > 0 { serving = "yes" } - fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n", - m.Name, serving, m.Ready, m.Min, modelStatus(m), + fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\n", + m.Name, serving, m.Ready, m.Min, modelStatus(m), reserves(m), join(m.Aliases), join(m.Profiles), join(m.Fallbacks)) + for _, x := range m.Warnings { + warnings = append(warnings, m.Name+": "+x) + } } - return w.Flush() + if err := w.Flush(); err != nil { + return err + } + for _, x := range warnings { + fmt.Println("warning:", x) + } + return nil }, } } @@ -511,3 +522,22 @@ func dash(s string) string { } return s } + +// reserves is what one replica of a model holds of a node, with what it was +// measured using beside it: "6G 2c (5.1G used)". +func reserves(m adminapi.ModelView) string { + if m.MemoryGB == 0 && m.CPUCores == 0 { + return "-" + } + out := fmt.Sprintf("%dG", m.MemoryGB) + if m.CPUCores > 0 { + out += fmt.Sprintf(" %dc", m.CPUCores) + } + if m.MeasuredMemoryGB > 0 { + out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB) + } + if len(m.Warnings) > 0 { + out += " !" + } + return out +} diff --git a/internal/cli/model.go b/internal/cli/model.go index e0346a3..fad3dbb 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -192,3 +192,44 @@ func newScaleCmd(client func() *ctlClient) *cobra.Command { "how long an on-demand model stays up unasked-for; zero takes the cluster default") return c } + +func newResourcesCmd(client func() *ctlClient) *cobra.Command { + var memory, cores int + var profile string + c := &cobra.Command{ + Use: "resources --memory GB [--cores N]", + Short: "Set what a model reserves of a node: memory, and CPU cores", + Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" + + "weights, KV cache and engine together. Placement reserves it on the node,\n" + + "so a node holds only as many replicas as it has memory for.\n\n" + + "--cores is the CPU cores a replica with no GPU computes on. They are\n" + + "reserved the same way, and become the engine's thread count, so two\n" + + "replicas on one node stop competing for the same cores. Zero reserves\n" + + "none and leaves the thread count to the node.\n\n" + + "Running replicas keep what they were started with; `llmgw ctl restart\n" + + "--model M` moves them to the new figures.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("memory") { + return fmt.Errorf("--memory is required") + } + body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores}) + if err != nil { + return err + } + var m catalog.Model + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil { + return err + } + for _, p := range m.Profiles { + fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores) + } + fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name) + return nil + }, + } + c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB") + c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none") + c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out") + return c +} diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index f5a452c..174913b 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -75,6 +75,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run") f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for") f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots") + f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits") f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use") return c } diff --git a/internal/control/adminapi/actor_test.go b/internal/control/adminapi/actor_test.go index 1452ce6..cbc355b 100644 --- a/internal/control/adminapi/actor_test.go +++ b/internal/control/adminapi/actor_test.go @@ -4,6 +4,8 @@ import ( "net/http/httptest" "strings" "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) func TestAClaimedActorIsOneShortName(t *testing.T) { @@ -22,3 +24,22 @@ func TestAClaimedActorIsOneShortName(t *testing.T) { } } } + +func TestAReservationThatPlacementCannotTrustIsSaid(t *testing.T) { + cpu := func(mem int) catalog.Model { + return catalog.Model{Name: "m", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: mem}}} + } + if w := reservationWarnings(cpu(0), 0); len(w) != 1 || !strings.Contains(w[0], "declares no memory") { + t.Errorf("no memory declared: %v", w) + } + if w := reservationWarnings(cpu(6), 8.2); len(w) != 1 || !strings.Contains(w[0], "uses 8.2 GB and reserves 6") { + t.Errorf("under-declared: %v", w) + } + if w := reservationWarnings(cpu(6), 6.4); len(w) != 0 { + t.Errorf("within a tenth is mapped weights, not a wrong figure: %v", w) + } + gpu := catalog.Model{Name: "g", Profiles: []catalog.Profile{{Name: "tp1", Engine: "vllm", GPUClass: "h100-80g"}}} + if w := reservationWarnings(gpu, 3); len(w) != 0 { + t.Errorf("a GPU model needs no host memory figure: %v", w) + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index cc817ff..ccc1ab5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -57,6 +57,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models", a.addModel) mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel) mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) + mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources) mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity) mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) mux.HandleFunc("GET /api/v1/usage", a.usageByScope) @@ -119,6 +120,8 @@ type NodeView struct { // what decides whether a CPU profile can be placed here, so it belongs // where someone looks when a model will not schedule. MemoryGB int `json:"memory_gb,omitempty"` + // CPUCount is the machine's CPUs, 0 when the agent did not report them. + CPUCount int `json:"cpu_count,omitempty"` // Engines are the inference engines installed here, as the agent found // them. Absent means the agent is older than the probe and said nothing, // which is not the same as having none — placement treats it that way too, @@ -270,6 +273,15 @@ type ModelView struct { Unplaceable bool `json:"unplaceable,omitempty"` Fallbacks []string `json:"fallbacks,omitempty"` Profiles []string `json:"profiles"` + // MemoryGB and CPUCores are what one replica reserves, from its first + // profile; MeasuredMemoryGB is the most one of its replicas uses now, 0 + // when none is measured. + MemoryGB int `json:"memory_gb,omitempty"` + CPUCores int `json:"cpu_cores,omitempty"` + MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"` + // Warnings are what makes its reservation wrong: nothing declared where + // placement needs a figure, or less declared than it uses. + Warnings []string `json:"warnings,omitempty"` } // ReplicaView is a replica as `ctl replicas` shows it. @@ -396,8 +408,8 @@ func (a *API) Nodes() []NodeView { ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr, State: n.State.String(), Labels: n.Labels, GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)), - MemoryGB: n.MemoryTotalMB / 1024, - Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features, + MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount, + Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features, Replicas: replicasOn[n.ID], } if free, total := n.LocalDisk(); total > 0 { @@ -497,6 +509,15 @@ func (a *API) Models() []ModelView { for _, h := range a.ctl.Reconciler().ModelHealth() { health[h.Model] = h } + // The most one replica of each model uses, summed over its shards. + measured := map[string]int64{} + for _, rep := range a.Replicas("") { + var b int64 + for _, sh := range rep.Shards { + b += sh.MemoryBytes + } + measured[rep.Model] = max(measured[rep.Model], b) + } out := make([]ModelView, 0) for _, m := range a.ctl.Catalog().Models() { h := health[m.Name] @@ -511,6 +532,11 @@ func (a *API) Models() []ModelView { for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) } + if len(m.Profiles) > 0 { + v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores + } + v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30) + v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB) out = append(out, v) } return out @@ -581,6 +607,15 @@ type Models interface { Add(ctx context.Context, m catalog.Model, by string) error Remove(ctx context.Context, name string) error Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) + Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) +} + +// ResourcesRequest sets what a model's profiles reserve of a node. +type ResourcesRequest struct { + // Profile is one profile's name; empty is every profile. + Profile string `json:"profile,omitempty"` + MemoryGB int `json:"memory_gb"` + CPUCores int `json:"cpu_cores"` } // Usage reports what each group of the tenancy tree spent. The store @@ -826,6 +861,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) { }) } +// setResources sets a model's reserved memory and cores. Placed replicas keep +// what they were started with until they are restarted. +func (a *API) setResources(w http.ResponseWriter, r *http.Request) { + if a.models == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + var req ResourcesRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, m) +} + func (a *API) removeModel(w http.ResponseWriter, r *http.Request) { if a.models == nil { writeError(w, http.StatusNotImplemented, "this control plane has no store") @@ -1307,3 +1362,40 @@ func actorOf(r *http.Request) string { } return b.String() } + +// NodeCPUs is the CPU counts of ready nodes with at least this much memory: +// where a replica needing it could go. +func (a *API) NodeCPUs(minMemoryGB int) []int { + var out []int + for _, n := range a.ctl.Inventory().Nodes() { + if n.State.String() != "ready" || n.CPUCount == 0 { + continue + } + if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB { + continue + } + out = append(out, n.CPUCount) + } + return out +} + +// reservationWarnings says what is wrong with what a model reserves. +// +// Resident memory includes the weights mapped from disk, which the kernel can +// take back, so it overstates what a model needs by the part of the weights +// not in use. A tenth over is let pass for that; more is a figure too low for +// placement to be trusted with. +func reservationWarnings(m catalog.Model, measuredGB float64) []string { + var out []string + for _, p := range m.Profiles { + if p.CPUOnly() && p.MinMemoryGB <= 0 { + out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+ + "(llmgw ctl model resources %s --memory GB)", p.Name, m.Name)) + } + } + if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 { + out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory", + measuredGB, m.Profiles[0].MinMemoryGB)) + } + return out +} diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 21cc382..b8dea81 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -258,6 +258,13 @@ type fakeModels struct { err error } +func (f *fakeModels) Resources(_ context.Context, name, profile string, mem, cores int, _ string) (catalog.Model, error) { + if f.err != nil { + return catalog.Model{}, f.err + } + return catalog.Model{Name: name, Profiles: []catalog.Profile{{Name: "p", MinMemoryGB: mem, CPUCores: cores}}}, nil +} + func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) { if f.err != nil { return catalog.Model{}, f.err diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 8bfba45..9af2a31 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -4,6 +4,7 @@ package agentsvc import ( "fmt" "maps" + "runtime" "time" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" @@ -27,6 +28,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { EngineVersions: r.GetEngineVersions(), EnginesProbed: r.GetEnginesProbed(), MemoryTotalMB: int(r.GetMemoryTotalMb()), + CPUCount: int(r.GetCpuCount()), } if h := r.GetHost(); h != nil { n.Host = discovery.Host{ @@ -90,6 +92,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr, Labels: labels, AgentVersion: agentVersion, MemoryTotalMb: int64(inv.MemoryTotalMB), + CpuCount: int32(runtime.NumCPU()), } for _, w := range inv.Weights { r.Weights = append(r.Weights, &llmgwv1.LocalWeights{ @@ -192,6 +195,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { Fabric: cmd.Fabric, ProfileFp: cmd.ProfileFP, ApiKey: cmd.Spec.APIKey, + CpuThreads: int32(cmd.Spec.Threads), } } @@ -213,6 +217,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration Fabric: m.GetFabric(), ProfileFP: m.GetProfileFp(), APIKey: m.GetApiKey(), + Threads: int(m.GetCpuThreads()), Profile: engines.Profile{ Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(), TP: int(p.GetTp()), PP: int(p.GetPp()), diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 2655532..68de0e8 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -345,8 +345,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { problems := hb.GetNewProblems() storage := hb.GetStorage() memMB := int(hb.GetMemoryTotalMb()) + cpus := int(hb.GetCpuCount()) if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && - !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() && + !hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() && !hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 && !hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil { return @@ -406,6 +407,12 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { "from_mb", n.MemoryTotalMB, "to_mb", memMB) n.MemoryTotalMB = memMB } + if cpus > 0 && cpus != n.CPUCount { + if n.CPUCount > 0 { + s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus) + } + n.CPUCount = cpus + } if hb.GetEnginesChanged() { vers := maps.Clone(hb.GetEngineVersions()) if !maps.Equal(vers, n.EngineVersions) { diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index b64277c..95ae8b0 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -74,6 +74,17 @@ type Profile struct { // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` + // CPUCores is the cores a replica of a profile with no GPU is given: + // reserved out of its node's CPUs by placement, as memory is, and what + // its engine computes on (llama.cpp's --threads). + // + // Reserved because an engine takes every core it is told about, and two + // on one node that each think they have all of them spin against each + // other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second + // to under one each. Zero reserves nothing and leaves the thread count to + // the node, which is every profile from before this. + CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"` + // MinDiskGB is the free space a node needs before it can hold these // weights. // @@ -380,6 +391,7 @@ func (m Model) PlacementProfiles() []placement.Profile { NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, MinDiskGB: p.MinDiskGB, + CPUCores: p.CPUCores, // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, @@ -475,3 +487,19 @@ func (p Profile) Fingerprint() string { sum := sha256.Sum256(b) return hex.EncodeToString(sum[:8]) } + +// SuggestCores is the cores to give a replica with no GPU on nodes of these +// CPU counts: half of the smallest, so two replicas share any of them without +// competing. Zero when no node says, which reserves nothing. +func SuggestCores(cpuCounts []int) int { + smallest := 0 + for _, c := range cpuCounts { + if c > 0 && (smallest == 0 || c < smallest) { + smallest = c + } + } + if smallest == 0 { + return 0 + } + return max(smallest/2, 1) +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 8bfcce5..dc92159 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -79,6 +79,9 @@ type Node struct { // it, which is treated as no constraint — the same as every node before // this field existed. MemoryTotalMB int `json:"memory_total_mb,omitempty"` + // CPUCount is the CPUs the agent may run on. Zero is an agent too old to + // say, which, like memory, constrains nothing. + CPUCount int `json:"cpu_count,omitempty"` // Weights are the model files this node can see, however they got there. // Empty means the node was not asked to look, which is not the same as @@ -373,6 +376,9 @@ type Allocation struct { // Zero for a profile that declared nothing, which constrains nothing and // is counted as nothing. MemoryGB int + // CPUCores is the cores this shard was given to compute on, for a shard + // with no GPU; zero for one that shares whatever is free. + CPUCores int } // Store holds the cluster inventory. It is safe for concurrent use. diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 68429e9..6d22a2d 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -91,6 +91,9 @@ type Request struct { // use. They default to 16384 and 4096, what four parallel slots divide. CtxPool int `json:"ctx_pool,omitempty"` ContextLength int `json:"context_length,omitempty"` + // CPUCores is what a replica computes on; zero takes half the smallest + // node it fits on. + CPUCores int `json:"cpu_cores,omitempty"` // By is who asked, for the catalog's record. By string `json:"-"` } @@ -278,13 +281,20 @@ func (s *Server) run(j *Job) { } s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis)) + cores := req.CPUCores + if cores <= 0 { + if nc, ok := s.deps.Fleet.(interface{ NodeCPUs(int) []int }); ok { + cores = catalog.SuggestCores(nc.NodeCPUs(memGB)) + } + } + // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, Profiles: []catalog.Profile{{ Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, - Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, + Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, }}, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, } diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 94c7621..dbdbf61 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -18,6 +18,7 @@ package modelsvc import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "sync" @@ -104,6 +105,16 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error { if s.Declared(m.Name) { return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name) } + // A model with no GPU is all host memory, and one that declares none + // fits everywhere as far as placement can tell — however many of it + // there already are. Required of what is added now; models stored before + // this still load, and say what they are missing. + for _, p := range m.Profiles { + if p.CPUOnly() && p.MinMemoryGB <= 0 { + return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+ + "placement could not tell when a node is full", m.Name, p.Name) + } + } if s.check != nil { if err := s.check(ctx, m); err != nil { return err @@ -189,36 +200,72 @@ func (s *Service) Remove(ctx context.Context, name string) error { return nil } -// Scale changes how many replicas a model runs, without a restart. -// -// Only the scaling block moves. Everything else about the model — its source, -// its profiles, the engine arguments somebody tuned — is read back from what -// was stored and written out again untouched, so changing a number cannot -// quietly become a redefinition. -// -// A model declared in configuration is refused, for the same reason Remove -// refuses one: the YAML is read on every start, so the new number would hold -// until the next restart and then silently revert. A cluster that disagrees -// with its own config file is worse than an error message saying where to edit. +// Scale changes how many replicas a model added at runtime runs. func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error { + m.Scaling = sc + return nil + }) + if err == nil { + s.log.Info("model scaled", "model", name, + "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by) + } + return final, err +} + +// Resources sets what a model's profile reserves of a node: host memory, and +// the CPU cores a replica without a GPU computes on. An empty profile name +// sets every profile. +func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) { + if memoryGB < 0 || cpuCores < 0 { + return catalog.Model{}, errors.New("memory and cores cannot be negative") + } + final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error { + found := false + for i := range m.Profiles { + if profile != "" && m.Profiles[i].Name != profile { + continue + } + found = true + m.Profiles[i].MinMemoryGB = memoryGB + m.Profiles[i].CPUCores = cpuCores + } + if !found { + return fmt.Errorf("model %q has no profile %q", name, profile) + } + return nil + }) + if err == nil { + s.log.Info("model resources set", "model", name, "profile", profile, + "memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by) + } + return final, err +} + +// update changes one model added at runtime and keeps the change. +// +// Built before anything is written, like Add: catalog.New is what applies the +// defaults and refuses what does not validate, so the model that comes back +// is the one that will run rather than the one that was asked for. +func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) { s.mu.Lock() defer s.mu.Unlock() if s.Declared(name) { return catalog.Model{}, fmt.Errorf( - "model %q is declared in configuration; change its scaling there, or it reverts on the next restart", name) + "model %q is declared in configuration; %s there, or it reverts on the next restart", name, what) } - stored, err := s.storedModels(ctx) if err != nil { return catalog.Model{}, err } - next := append([]catalog.Model{}, s.declared...) found := false for _, e := range stored { if e.Name == name { - e.Scaling = sc + if err := change(&e); err != nil { + return catalog.Model{}, err + } found = true } next = append(next, e) @@ -226,20 +273,14 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by if !found { return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name) } - - // Built before anything is written, like Add: catalog.New is what applies - // the defaults (max follows min when it is left out) and what refuses a - // max below a min, so the model that comes back is the one that will run - // rather than the one that was asked for. built, err := catalog.New(next) if err != nil { - return catalog.Model{}, fmt.Errorf("scaling %q that way would make the catalog invalid: %w", name, err) + return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err) } final, ok := built.Resolve(name) if !ok { return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name) } - def, err := json.Marshal(final) if err != nil { return catalog.Model{}, err @@ -249,10 +290,7 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by }); err != nil { return catalog.Model{}, err } - s.apply.ReplaceCatalog(ctx, built) - s.log.Info("model scaled", "model", name, - "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by) return final, nil } diff --git a/internal/control/modelsvc/modelsvc_test.go b/internal/control/modelsvc/modelsvc_test.go index 63b49dd..bd72e95 100644 --- a/internal/control/modelsvc/modelsvc_test.go +++ b/internal/control/modelsvc/modelsvc_test.go @@ -48,7 +48,7 @@ func model(name string) catalog.Model { return catalog.Model{ Name: name, Source: "/models/" + name, Scaling: catalog.Scaling{Min: 1, Max: 1}, - Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}}, + Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1, MinMemoryGB: 2}}, } } @@ -337,3 +337,43 @@ func TestMinZeroWithAMaxIsOnDemand(t *testing.T) { t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout) } } + +func TestACPUModelAddedNowMustSayHowMuchMemoryItNeeds(t *testing.T) { + s, st, _ := newService(t) + m := model("bare") + m.Profiles[0].MinMemoryGB = 0 + err := s.Add(t.Context(), m, "tester") + if err == nil || !strings.Contains(err.Error(), "declares no memory") { + t.Fatalf("a CPU model with no memory was added: %v", err) + } + if len(st.rows) != 0 { + t.Error("it was stored anyway") + } +} + +func TestResourcesAreSetOnAStoredModelAndSurviveARestart(t *testing.T) { + s, _, _ := newService(t) + if err := s.Add(t.Context(), model("m"), "tester"); err != nil { + t.Fatal(err) + } + got, err := s.Resources(t.Context(), "m", "", 6, 3, "tester") + if err != nil { + t.Fatal(err) + } + if p := got.Profiles[0]; p.MinMemoryGB != 6 || p.CPUCores != 3 { + t.Errorf("profile %+v", p) + } + if _, err := s.Resources(t.Context(), "m", "nope", 6, 3, "tester"); err == nil { + t.Error("a profile the model does not have was accepted") + } + if _, err := s.Resources(t.Context(), "m", "", -1, 0, "tester"); err == nil { + t.Error("negative memory was accepted") + } + cat, err := s.Load(t.Context()) + if err != nil { + t.Fatal(err) + } + if m, _ := cat.Resolve("m"); m.Profiles[0].CPUCores != 3 { + t.Error("the cores were lost on reload") + } +} diff --git a/internal/control/placement/cpu_test.go b/internal/control/placement/cpu_test.go index d5c4891..5f72dc1 100644 --- a/internal/control/placement/cpu_test.go +++ b/internal/control/placement/cpu_test.go @@ -1,6 +1,7 @@ package placement import ( + "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" @@ -100,3 +101,38 @@ func TestCPUProfileRespectsAvoid(t *testing.T) { t.Errorf("placed on %q despite being told to avoid it", got) } } + +func TestCPUCoresAreReservedAsMemoryIs(t *testing.T) { + n := node("n1", "", 0) + n.CPUCount = 8 + st := store(n) + s := New(st) + p := cpuProfile("cpu") + p.CPUCores = 3 + + for i := range 2 { + res := s.Place([]Profile{p}, nil) + if res.Placement == nil { + t.Fatalf("replica %d did not fit in 8 cores:\n%s", i+1, res.Explain()) + } + st.Allocate("n1", inventory.Allocation{ShardID: string(rune('a' + i)), CPUCores: 3}) + } + // Six of eight given away: a third would need three. + res := s.Place([]Profile{p}, nil) + if res.Placement != nil { + t.Fatal("a third replica was given cores the node does not have") + } + if got := res.Explain(); !strings.Contains(got, "has 8 CPUs with 2 free, needs 3") { + t.Errorf("the refusal should say how many cores are left:\n%s", got) + } + + // Unknown counts constrain nothing: an older agent, or a profile that + // reserves no cores. + old := node("n2", "", 0) + if res := New(store(old)).Place([]Profile{p}, nil); res.Placement == nil { + t.Error("a node that does not report its CPUs was excluded") + } + if res := s.Place([]Profile{cpuProfile("cpu")}, nil); res.Placement == nil { + t.Error("a profile reserving no cores was refused") + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 59ee579..fdb92d7 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -39,6 +39,9 @@ type Profile struct { // MinDiskGB is the free space needed to hold the weights, checked only on // nodes that do not already have them. Zero is unconstrained. MinDiskGB int + // CPUCores is the cores a profile with no GPU reserves. Zero is + // unconstrained. + CPUCores int // Source is the model's weights, as the engine would be given it. Used to // prefer nodes that already hold them; empty asks for no preference. Source string @@ -582,6 +585,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje } } + // Cores, for a replica that computes on them: what is left after the + // cores already given to the node's other replicas. Same exemptions + // as memory: zero declares nothing, and a node that does not report + // its CPUs is an older agent, not one with none. + if p.CPUOnly() && p.CPUCores > 0 && n.CPUCount > 0 { + free := n.CPUCount - s.claimedCores(n.ID) + if free < p.CPUCores { + reject(fmt.Sprintf("has %d CPUs with %d free, needs %d", n.CPUCount, free, p.CPUCores)) + continue + } + } + if p.CPUOnly() { // Every ready node can run it; there is no card to be free. out = append(out, candidate{node: n}) @@ -723,6 +738,15 @@ func (s *Scheduler) claimedMemory(nodeID string) int { return total } +// claimedCores is the CPU cores already given to replicas on a node. +func (s *Scheduler) claimedCores(nodeID string) int { + total := 0 + for _, a := range s.inv.Allocations(nodeID) { + total += a.CPUCores + } + return total +} + // installing says why an engine is missing, when the node has said. func installing(n inventory.Node, engine string) string { for _, p := range n.Provision { diff --git a/internal/control/reconciler/adopt.go b/internal/control/reconciler/adopt.go index 8a27c38..b5830e3 100644 --- a/internal/control/reconciler/adopt.go +++ b/internal/control/reconciler/adopt.go @@ -144,6 +144,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) { r.inv.Allocate(s.NodeID, inventory.Allocation{ ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices, MemoryGB: r.profileMemory(rep.Model, rep.Profile), + CPUCores: r.adoptedCores(rep.Model, rep.Profile), }) // The endpoint comes from the leader, which may be in any of the // groups — the rank 0 node is not necessarily the first to register. @@ -253,3 +254,9 @@ type AdoptedShard struct { // Endpoint is the leader's base URL, set on the rank 0 shard. Endpoint string } + +// adoptedCores is what an adopted shard is taken to hold of its node's CPUs. +func (r *Reconciler) adoptedCores(model, profile string) int { + p, _ := r.profileOf(model, profile) + return cpuCores(p) +} diff --git a/internal/control/reconciler/headroom.go b/internal/control/reconciler/headroom.go index fae46a5..6080b03 100644 --- a/internal/control/reconciler/headroom.go +++ b/internal/control/reconciler/headroom.go @@ -66,14 +66,14 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { h.Blocked = slices.Compact(h.Blocked) return h, true } - mem := 0 + mem, cores := 0, 0 if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok { - mem = prof.MinMemoryGB + mem, cores = prof.MinMemoryGB, cpuCores(prof) } var nodes []string for _, a := range res.Placement.Nodes { inv.Allocate(a.NodeID, inventory.Allocation{ - ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, + ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, }) nodes = append(nodes, a.NodeID) } diff --git a/internal/control/reconciler/headroom_test.go b/internal/control/reconciler/headroom_test.go index 0bf3c1d..44f9111 100644 --- a/internal/control/reconciler/headroom_test.go +++ b/internal/control/reconciler/headroom_test.go @@ -57,3 +57,24 @@ func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) { t.Errorf("headroom %+v, expected room for 3 more", got) } } + +func TestReservedCoresAreTheEnginesThreadsAndBoundTheRoom(t *testing.T) { + cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 64 * 1024, CPUCount: 8} + m := catalog.Model{Name: "coder", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6, CPUCores: 3}}} + h := newHarness(t, m, cpu) + h.r.Reconcile(t.Context()) + + started := h.disp.startedShards() + if len(started) != 1 || started[0].Spec.Threads != 3 { + t.Fatalf("the engine was not told its 3 cores: %+v", started) + } + if a := h.inv.Allocations("a"); len(a) != 1 || a[0].CPUCores != 3 { + t.Errorf("the cores were not reserved: %+v", a) + } + // Memory would allow nine more; cores allow one: 8 - 3 = 5, then 2. + got, _ := h.r.Headroom("coder", 10) + if got.More != 1 { + t.Errorf("room for %d more, expected 1 by cores: %+v", got.More, got) + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 9f86bd6..8572f08 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -825,7 +825,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac }) r.inv.Allocate(a.NodeID, inventory.Allocation{ ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices, - MemoryGB: prof.MinMemoryGB, + MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), }) } rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort) @@ -869,6 +869,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac HTTPPort: s.HTTPPort, Profile: espec, APIKey: r.engineKey(id), + Threads: cpuCores(prof), }, StartupTimeout: prof.StartupTimeout, Pool: pl.Pool, @@ -1108,16 +1109,31 @@ func addrOf(n inventory.Node) string { // claim on restart would place a second large model onto a node already full of // the first. func (r *Reconciler) profileMemory(model, profile string) int { + p, _ := r.profileOf(model, profile) + return p.MinMemoryGB +} + +// profileOf is a replica's profile as the catalog holds it now. +func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) { m, ok := r.cat.Load().Resolve(model) if !ok { - return 0 + return catalog.Profile{}, false } for _, p := range m.Profiles { if p.Name == profile { - return p.MinMemoryGB + return p, true } } - return 0 + return catalog.Profile{}, false +} + +// cpuCores is what a profile reserves of a node's CPUs: its cores when it +// computes on them, nothing when a GPU does the work. +func cpuCores(p catalog.Profile) int { + if !p.CPUOnly() { + return 0 + } + return p.CPUCores } func (r *Reconciler) engineKey(replicaID string) string { diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 98cfa1e..4ac7666 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -49,6 +49,9 @@ type serveForm struct { // 3.5 GB for a KV pool. Nothing was wrong with placement — it was never // told what the model needed. MinMemoryGB int + // CPUCores is the cores a replica computes on: half the smallest node it + // fits on, or zero when no node says how many it has. + CPUCores int // MemoryBasis says how MinMemoryGB was arrived at, for the form to show. MemoryBasis string // CtxPool is the engine context pool the form starts with, and the one @@ -152,6 +155,15 @@ func planServe(st mirror.Status, files []mirror.Status, base string, defaultCtxPool, header.BlockCount, kvHeads, float64(int64(1<<30)+weights/20)/(1<<30)) } + if fleet != nil { + var cpus []int + for _, n := range fleet.Nodes() { + if n.State == "ready" && (n.MemoryGB == 0 || n.MemoryGB >= f.MinMemoryGB) { + cpus = append(cpus, n.CPUCount) + } + } + f.CPUCores = catalog.SuggestCores(cpus) + } if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest { f.Warning = strings.TrimSuffix(f.Warning, ".") + fmt.Sprintf("%sthe largest node has %d GB and this needs about %d", @@ -277,6 +289,11 @@ func buildModel(get func(string) string) (catalog.Model, error) { return m, err } } + if v := strings.TrimSpace(get("cpu_cores")); v != "" { + if p.CPUCores, err = positive(v, "the cores"); err != nil { + return m, err + } + } if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" { // What keeps a large model off the small nodes. A CPU profile has no // card to be short of, so without this every ready node looks like a diff --git a/internal/web/static/base.css b/internal/web/static/base.css index b98f799..489bc2e 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -854,3 +854,4 @@ table.explore td:first-child { max-width: 34rem; overflow: hidden; text-overflow table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted); } .clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis; vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; } +table.rows td.warn { color: var(--warn); cursor: help; } diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index a97349c..42aba77 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -165,7 +165,9 @@ reachable from this machine.

+ + diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 116621d..026a9a6 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -80,12 +80,14 @@

Models

{{if .Models}} -{{if .AnyFallbacks}}{{end}} +
ModelAliasesReadyWantedProfilesFallbacksWhat it is for
{{if .AnyFallbacks}}{{end}} {{range .Models}} + {{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}} + {{- if .MeasuredMemoryGB}} {{printf "%.1f" .MeasuredMemoryGB}}G used{{end}}{{if .Warnings}} ⚠{{end}} {{if $.AnyFallbacks}}{{end}} {{/* Cut to the column, the whole sentence on hover: a row of its own for
ModelAliasesReadyWantedReservesProfilesFallbacksWhat it is for
{{.Name}} {{join .Aliases}} {{.Ready}} {{.Min}}–{{.Max}}{{join .Profiles}}{{join .Fallbacks}}