diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4ddf293..3f593e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,30 @@ APIs change between minors without a deprecation period.
Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until
there are enough for v0.10.5.
+### CPU cores are reserved, and memory has to be declared
+
+The first step of resource allocation for larger clusters: what a replica
+holds of a node is reserved, not assumed.
+
+- **Cores.** A profile's `cpu_cores` is reserved out of its node's CPUs, as
+ memory is, and is what its engine computes on (llama.cpp's `--threads`). A
+ node holds as many CPU replicas as it has cores for, and two on one node no
+ longer spin against each other. Agents report their CPU count; a node that
+ does not, and a profile that declares no cores, constrain nothing.
+- **Memory is required of new CPU models.** A model with no GPU is all host
+ memory, and one declaring none looked to placement as if it fitted
+ everywhere, any number of times. Adding one without `min_memory_gb` is now
+ refused. Models stored before this still load, and say what is missing.
+- **Declared against measured.** Each model shows what one replica reserves
+ and the most one of its replicas uses — in `ctl models` (RESERVES) and on the
+ cluster page — with a warning when nothing is declared or when it uses more
+ than a tenth over what it reserves.
+- **Setting them.** `llmgw ctl model resources MODEL --memory GB --cores N`
+ (and `POST /api/v1/models/{name}/resources`) for a model added at runtime.
+ `ctl model serve` and the mirror's serve form fill in cores — half the
+ smallest node the model fits on — beside the memory they already estimate.
+ Running replicas keep their old figures until restarted.
+
### The mirror page in tabs
/admin/mirror was one long page: what the mirror holds, the Hugging Face
diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go
index b0ef34a..41adf67 100644
--- a/api/gen/llmgw/v1/agent.pb.go
+++ b/api/gen/llmgw/v1/agent.pb.go
@@ -534,6 +534,10 @@ type Register struct {
// meant "an agent too old to say" and constrained nothing — so a freshly
// built node with no engine installed yet was a candidate for every model.
EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"`
+ // cpu_count is the CPUs the agent may run on. Placement reserves cores
+ // for CPU replicas out of it, the way it reserves memory out of
+ // memory_total_mb; zero, from an older agent, reserves nothing.
+ CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
// imex is the service that lets a multi-node NVLink domain share memory.
Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"`
// installable_engines are the engines this agent knows how to install
@@ -679,6 +683,13 @@ func (x *Register) GetEnginesProbed() bool {
return false
}
+func (x *Register) GetCpuCount() int32 {
+ if x != nil {
+ return x.CpuCount
+ }
+ return 0
+}
+
func (x *Register) GetImex() *IMEX {
if x != nil {
return x.Imex
@@ -2017,8 +2028,10 @@ type Heartbeat struct {
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
// once a minute: a fabric manager restarting drops every card out of its
// domain, and the node has to stop being offered work that spans it.
- GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
- Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
+ GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
+ Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
+ // cpu_count as in Register, for a VM given more CPUs while it runs.
+ CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -2172,6 +2185,13 @@ func (x *Heartbeat) GetImex() *IMEX {
return nil
}
+func (x *Heartbeat) GetCpuCount() int32 {
+ if x != nil {
+ return x.CpuCount
+ }
+ return 0
+}
+
// Provision is the software a node should have: which engines, at which
// versions. Nothing more.
//
@@ -2561,7 +2581,10 @@ type StartShard struct {
// api_key is what the engine demands of every request, derived by the
// control plane from the replica id. The router sends it; nothing else
// that can reach the node's ports has it.
- ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
+ ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
+ // cpu_threads is how many CPU threads a shard with no GPU computes on:
+ // the cores placement reserved for it. Zero leaves it to the agent.
+ CpuThreads int32 `protobuf:"varint,17,opt,name=cpu_threads,json=cpuThreads,proto3" json:"cpu_threads,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -2708,6 +2731,13 @@ func (x *StartShard) GetApiKey() string {
return ""
}
+func (x *StartShard) GetCpuThreads() int32 {
+ if x != nil {
+ return x.CpuThreads
+ }
+ return 0
+}
+
type EngineProfile struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
@@ -3655,7 +3685,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"fabricTest\x122\n" +
"\aupgrade\x18\n" +
" \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" +
- "\x03msg\"\xe3\x06\n" +
+ "\x03msg\"\x80\a\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -3671,7 +3701,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" +
"\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" +
"\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" +
- "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" +
+ "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\x1b\n" +
+ "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x12\"\n" +
"\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" +
"\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x12\x1a\n" +
"\bfeatures\x18\x11 \x03(\tR\bfeatures\x1a9\n" +
@@ -3811,7 +3842,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x06status\x18\x03 \x01(\tR\x06status\x12!\n" +
"\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" +
"\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" +
- "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd9\x06\n" +
+ "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xf6\x06\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -3831,7 +3862,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x127\n" +
"\n" +
"gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" +
- "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1aA\n" +
+ "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" +
+ "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
@@ -3872,7 +3904,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" +
"\vbytes_total\x18\x03 \x01(\x03R\n" +
- "bytesTotal\"\xf2\x03\n" +
+ "bytesTotal\"\x93\x04\n" +
"\n" +
"StartShard\x12\x1d\n" +
"\n" +
@@ -3896,7 +3928,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" +
"\n" +
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
- "\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" +
+ "\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" +
+ "\vcpu_threads\x18\x11 \x01(\x05R\n" +
+ "cpuThreads\"\xfd\x02\n" +
"\rEngineProfile\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto
index 811f095..cd24d04 100644
--- a/api/proto/llmgw/v1/agent.proto
+++ b/api/proto/llmgw/v1/agent.proto
@@ -120,6 +120,10 @@ message Register {
// meant "an agent too old to say" and constrained nothing — so a freshly
// built node with no engine installed yet was a candidate for every model.
bool engines_probed = 14;
+ // cpu_count is the CPUs the agent may run on. Placement reserves cores
+ // for CPU replicas out of it, the way it reserves memory out of
+ // memory_total_mb; zero, from an older agent, reserves nothing.
+ int32 cpu_count = 18;
// imex is the service that lets a multi-node NVLink domain share memory.
IMEX imex = 15;
@@ -420,6 +424,8 @@ message Heartbeat {
// domain, and the node has to stop being offered work that spans it.
repeated GPUFabricState gpu_fabric = 16;
IMEX imex = 17;
+ // cpu_count as in Register, for a VM given more CPUs while it runs.
+ int32 cpu_count = 18;
}
// Provision is the software a node should have: which engines, at which
@@ -530,6 +536,9 @@ message StartShard {
// control plane from the replica id. The router sends it; nothing else
// that can reach the node's ports has it.
string api_key = 16;
+ // cpu_threads is how many CPU threads a shard with no GPU computes on:
+ // the cores placement reserved for it. Zero leaves it to the agent.
+ int32 cpu_threads = 17;
}
message EngineProfile {
diff --git a/docs/reference.md b/docs/reference.md
index bce4b1a..a1199f3 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -129,6 +129,7 @@ rather than repeated.
| `args` | `[]string` | [Profile](#profile) |
| `env` | `map[string]string` | [Profile](#profile) |
| `min_memory_gb` | `int` | [Profile](#profile) |
+| `cpu_cores` | `int` | [Profile](#profile) |
| `min_disk_gb` | `int` | [Profile](#profile) |
| `node_selector` | `map[string]string` | [Profile](#profile) |
| `weight` | `float64` | [Profile](#profile) |
@@ -966,6 +967,20 @@ this. A node that does not report its memory is not excluded either —
an older agent must not have its nodes quietly emptied by an upgrade of
the control plane.
+### `cpu_cores`
+
+`int`
+
+CPUCores is the cores a replica of a profile with no GPU is given:
+reserved out of its node's CPUs by placement, as memory is, and what
+its engine computes on (llama.cpp's --threads).
+
+Reserved because an engine takes every core it is told about, and two
+on one node that each think they have all of them spin against each
+other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
+to under one each. Zero reserves nothing and leaves the thread count to
+the node, which is every profile from before this.
+
### `min_disk_gb`
`int`
diff --git a/internal/agent/agent.go b/internal/agent/agent.go
index 4801fa5..72afbf3 100644
--- a/internal/agent/agent.go
+++ b/internal/agent/agent.go
@@ -16,6 +16,7 @@ import (
"net/http"
"os"
"path/filepath"
+ "runtime"
"slices"
"sync"
"time"
@@ -549,6 +550,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
// and a node that grew from 7 GB to 32 GB has to be seen to have grown
// without anybody restarting anything.
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
+ hb.CpuCount = int32(runtime.NumCPU())
a.reprobeEngines()
a.mu.Lock()
diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go
index 4f7862a..52569bf 100644
--- a/internal/agent/engines/engine.go
+++ b/internal/agent/engines/engine.go
@@ -76,6 +76,10 @@ type ShardSpec struct {
// and accounting. Given to the engine in its environment rather than on
// its command line, which every user on the node can read.
APIKey string `json:"api_key,omitempty"`
+ // Threads is the CPU cores placement reserved for a shard with no GPU,
+ // and so what its engine computes on. Zero leaves it to the engine's own
+ // configuration.
+ Threads int `json:"threads,omitempty"`
// Pool and Fabric are control plane concepts the agent cannot derive. They
// are carried so the agent can report them back after a restart; Fabric
diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go
index 7b500af..4de8060 100644
--- a/internal/agent/engines/llamacpp.go
+++ b/internal/agent/engines/llamacpp.go
@@ -133,7 +133,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
args = append(args, "--n-gpu-layers", "999")
}
if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
- n := l.cfg.Threads
+ // What placement reserved first: it is what the node's other
+ // replicas were placed around.
+ n := s.Threads
+ if n <= 0 {
+ n = l.cfg.Threads
+ }
if n <= 0 {
n = max(numCPU()/2, 1)
}
diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go
index aff2d69..3cafd35 100644
--- a/internal/agent/engines/llamacpp_test.go
+++ b/internal/agent/engines/llamacpp_test.go
@@ -294,3 +294,15 @@ func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
t.Errorf("--threads %q on one CPU", v)
}
}
+
+func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
+ s := llamaSpec(nil)
+ s.Threads = 3
+ cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
+ t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
+ }
+}
diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go
index 45a6d3e..aadc972 100644
--- a/internal/cli/ctl.go
+++ b/internal/cli/ctl.go
@@ -139,6 +139,7 @@ func newCtlCmd() *cobra.Command {
newStatusCmd(client, &asJSON),
newModelCmd(client),
newScaleCmd(client),
+ newResourcesCmd(client),
newRestartCmd(client),
newRetryCmd(client),
newUpgradeCmd(client),
@@ -323,17 +324,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
- fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS")
+ fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
+ var warnings []string
for _, m := range models {
serving := "no"
if m.Ready > 0 {
serving = "yes"
}
- fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n",
- m.Name, serving, m.Ready, m.Min, modelStatus(m),
+ fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\n",
+ m.Name, serving, m.Ready, m.Min, modelStatus(m), reserves(m),
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
+ for _, x := range m.Warnings {
+ warnings = append(warnings, m.Name+": "+x)
+ }
}
- return w.Flush()
+ if err := w.Flush(); err != nil {
+ return err
+ }
+ for _, x := range warnings {
+ fmt.Println("warning:", x)
+ }
+ return nil
},
}
}
@@ -511,3 +522,22 @@ func dash(s string) string {
}
return s
}
+
+// reserves is what one replica of a model holds of a node, with what it was
+// measured using beside it: "6G 2c (5.1G used)".
+func reserves(m adminapi.ModelView) string {
+ if m.MemoryGB == 0 && m.CPUCores == 0 {
+ return "-"
+ }
+ out := fmt.Sprintf("%dG", m.MemoryGB)
+ if m.CPUCores > 0 {
+ out += fmt.Sprintf(" %dc", m.CPUCores)
+ }
+ if m.MeasuredMemoryGB > 0 {
+ out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB)
+ }
+ if len(m.Warnings) > 0 {
+ out += " !"
+ }
+ return out
+}
diff --git a/internal/cli/model.go b/internal/cli/model.go
index e0346a3..fad3dbb 100644
--- a/internal/cli/model.go
+++ b/internal/cli/model.go
@@ -192,3 +192,44 @@ func newScaleCmd(client func() *ctlClient) *cobra.Command {
"how long an on-demand model stays up unasked-for; zero takes the cluster default")
return c
}
+
+func newResourcesCmd(client func() *ctlClient) *cobra.Command {
+ var memory, cores int
+ var profile string
+ c := &cobra.Command{
+ Use: "resources --memory GB [--cores N]",
+ Short: "Set what a model reserves of a node: memory, and CPU cores",
+ Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" +
+ "weights, KV cache and engine together. Placement reserves it on the node,\n" +
+ "so a node holds only as many replicas as it has memory for.\n\n" +
+ "--cores is the CPU cores a replica with no GPU computes on. They are\n" +
+ "reserved the same way, and become the engine's thread count, so two\n" +
+ "replicas on one node stop competing for the same cores. Zero reserves\n" +
+ "none and leaves the thread count to the node.\n\n" +
+ "Running replicas keep what they were started with; `llmgw ctl restart\n" +
+ "--model M` moves them to the new figures.",
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ if !cmd.Flags().Changed("memory") {
+ return fmt.Errorf("--memory is required")
+ }
+ body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores})
+ if err != nil {
+ return err
+ }
+ var m catalog.Model
+ if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil {
+ return err
+ }
+ for _, p := range m.Profiles {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores)
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name)
+ return nil
+ },
+ }
+ c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB")
+ c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none")
+ c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
+ return c
+}
diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go
index f5a452c..174913b 100644
--- a/internal/cli/model_serve.go
+++ b/internal/cli/model_serve.go
@@ -75,6 +75,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
+ f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
return c
}
diff --git a/internal/control/adminapi/actor_test.go b/internal/control/adminapi/actor_test.go
index 1452ce6..cbc355b 100644
--- a/internal/control/adminapi/actor_test.go
+++ b/internal/control/adminapi/actor_test.go
@@ -4,6 +4,8 @@ import (
"net/http/httptest"
"strings"
"testing"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func TestAClaimedActorIsOneShortName(t *testing.T) {
@@ -22,3 +24,22 @@ func TestAClaimedActorIsOneShortName(t *testing.T) {
}
}
}
+
+func TestAReservationThatPlacementCannotTrustIsSaid(t *testing.T) {
+ cpu := func(mem int) catalog.Model {
+ return catalog.Model{Name: "m", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: mem}}}
+ }
+ if w := reservationWarnings(cpu(0), 0); len(w) != 1 || !strings.Contains(w[0], "declares no memory") {
+ t.Errorf("no memory declared: %v", w)
+ }
+ if w := reservationWarnings(cpu(6), 8.2); len(w) != 1 || !strings.Contains(w[0], "uses 8.2 GB and reserves 6") {
+ t.Errorf("under-declared: %v", w)
+ }
+ if w := reservationWarnings(cpu(6), 6.4); len(w) != 0 {
+ t.Errorf("within a tenth is mapped weights, not a wrong figure: %v", w)
+ }
+ gpu := catalog.Model{Name: "g", Profiles: []catalog.Profile{{Name: "tp1", Engine: "vllm", GPUClass: "h100-80g"}}}
+ if w := reservationWarnings(gpu, 3); len(w) != 0 {
+ t.Errorf("a GPU model needs no host memory figure: %v", w)
+ }
+}
diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go
index cc817ff..ccc1ab5 100644
--- a/internal/control/adminapi/api.go
+++ b/internal/control/adminapi/api.go
@@ -57,6 +57,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models", a.addModel)
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
+ mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
@@ -119,6 +120,8 @@ type NodeView struct {
// what decides whether a CPU profile can be placed here, so it belongs
// where someone looks when a model will not schedule.
MemoryGB int `json:"memory_gb,omitempty"`
+ // CPUCount is the machine's CPUs, 0 when the agent did not report them.
+ CPUCount int `json:"cpu_count,omitempty"`
// Engines are the inference engines installed here, as the agent found
// them. Absent means the agent is older than the probe and said nothing,
// which is not the same as having none — placement treats it that way too,
@@ -270,6 +273,15 @@ type ModelView struct {
Unplaceable bool `json:"unplaceable,omitempty"`
Fallbacks []string `json:"fallbacks,omitempty"`
Profiles []string `json:"profiles"`
+ // MemoryGB and CPUCores are what one replica reserves, from its first
+ // profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
+ // when none is measured.
+ MemoryGB int `json:"memory_gb,omitempty"`
+ CPUCores int `json:"cpu_cores,omitempty"`
+ MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"`
+ // Warnings are what makes its reservation wrong: nothing declared where
+ // placement needs a figure, or less declared than it uses.
+ Warnings []string `json:"warnings,omitempty"`
}
// ReplicaView is a replica as `ctl replicas` shows it.
@@ -396,8 +408,8 @@ func (a *API) Nodes() []NodeView {
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
State: n.State.String(), Labels: n.Labels,
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
- MemoryGB: n.MemoryTotalMB / 1024,
- Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
+ MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount,
+ Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
Replicas: replicasOn[n.ID],
}
if free, total := n.LocalDisk(); total > 0 {
@@ -497,6 +509,15 @@ func (a *API) Models() []ModelView {
for _, h := range a.ctl.Reconciler().ModelHealth() {
health[h.Model] = h
}
+ // The most one replica of each model uses, summed over its shards.
+ measured := map[string]int64{}
+ for _, rep := range a.Replicas("") {
+ var b int64
+ for _, sh := range rep.Shards {
+ b += sh.MemoryBytes
+ }
+ measured[rep.Model] = max(measured[rep.Model], b)
+ }
out := make([]ModelView, 0)
for _, m := range a.ctl.Catalog().Models() {
h := health[m.Name]
@@ -511,6 +532,11 @@ func (a *API) Models() []ModelView {
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
}
+ if len(m.Profiles) > 0 {
+ v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
+ }
+ v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
+ v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
out = append(out, v)
}
return out
@@ -581,6 +607,15 @@ type Models interface {
Add(ctx context.Context, m catalog.Model, by string) error
Remove(ctx context.Context, name string) error
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
+ Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
+}
+
+// ResourcesRequest sets what a model's profiles reserve of a node.
+type ResourcesRequest struct {
+ // Profile is one profile's name; empty is every profile.
+ Profile string `json:"profile,omitempty"`
+ MemoryGB int `json:"memory_gb"`
+ CPUCores int `json:"cpu_cores"`
}
// Usage reports what each group of the tenancy tree spent. The store
@@ -826,6 +861,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
})
}
+// setResources sets a model's reserved memory and cores. Placed replicas keep
+// what they were started with until they are restarted.
+func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
+ if a.models == nil {
+ writeError(w, http.StatusNotImplemented, "this control plane has no store")
+ return
+ }
+ var req ResourcesRequest
+ if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
+ writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
+ return
+ }
+ m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r))
+ if err != nil {
+ writeError(w, http.StatusBadRequest, err.Error())
+ return
+ }
+ writeJSON(w, http.StatusOK, m)
+}
+
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
@@ -1307,3 +1362,40 @@ func actorOf(r *http.Request) string {
}
return b.String()
}
+
+// NodeCPUs is the CPU counts of ready nodes with at least this much memory:
+// where a replica needing it could go.
+func (a *API) NodeCPUs(minMemoryGB int) []int {
+ var out []int
+ for _, n := range a.ctl.Inventory().Nodes() {
+ if n.State.String() != "ready" || n.CPUCount == 0 {
+ continue
+ }
+ if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB {
+ continue
+ }
+ out = append(out, n.CPUCount)
+ }
+ return out
+}
+
+// reservationWarnings says what is wrong with what a model reserves.
+//
+// Resident memory includes the weights mapped from disk, which the kernel can
+// take back, so it overstates what a model needs by the part of the weights
+// not in use. A tenth over is let pass for that; more is a figure too low for
+// placement to be trusted with.
+func reservationWarnings(m catalog.Model, measuredGB float64) []string {
+ var out []string
+ for _, p := range m.Profiles {
+ if p.CPUOnly() && p.MinMemoryGB <= 0 {
+ out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+
+ "(llmgw ctl model resources %s --memory GB)", p.Name, m.Name))
+ }
+ }
+ if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 {
+ out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory",
+ measuredGB, m.Profiles[0].MinMemoryGB))
+ }
+ return out
+}
diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go
index 21cc382..b8dea81 100644
--- a/internal/control/adminapi/api_test.go
+++ b/internal/control/adminapi/api_test.go
@@ -258,6 +258,13 @@ type fakeModels struct {
err error
}
+func (f *fakeModels) Resources(_ context.Context, name, profile string, mem, cores int, _ string) (catalog.Model, error) {
+ if f.err != nil {
+ return catalog.Model{}, f.err
+ }
+ return catalog.Model{Name: name, Profiles: []catalog.Profile{{Name: "p", MinMemoryGB: mem, CPUCores: cores}}}, nil
+}
+
func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) {
if f.err != nil {
return catalog.Model{}, f.err
diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go
index 8bfba45..9af2a31 100644
--- a/internal/control/agentsvc/convert.go
+++ b/internal/control/agentsvc/convert.go
@@ -4,6 +4,7 @@ package agentsvc
import (
"fmt"
"maps"
+ "runtime"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
@@ -27,6 +28,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
EngineVersions: r.GetEngineVersions(),
EnginesProbed: r.GetEnginesProbed(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
+ CPUCount: int(r.GetCpuCount()),
}
if h := r.GetHost(); h != nil {
n.Host = discovery.Host{
@@ -90,6 +92,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr,
Labels: labels, AgentVersion: agentVersion,
MemoryTotalMb: int64(inv.MemoryTotalMB),
+ CpuCount: int32(runtime.NumCPU()),
}
for _, w := range inv.Weights {
r.Weights = append(r.Weights, &llmgwv1.LocalWeights{
@@ -192,6 +195,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
Fabric: cmd.Fabric,
ProfileFp: cmd.ProfileFP,
ApiKey: cmd.Spec.APIKey,
+ CpuThreads: int32(cmd.Spec.Threads),
}
}
@@ -213,6 +217,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
Fabric: m.GetFabric(),
ProfileFP: m.GetProfileFp(),
APIKey: m.GetApiKey(),
+ Threads: int(m.GetCpuThreads()),
Profile: engines.Profile{
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
TP: int(p.GetTp()), PP: int(p.GetPp()),
diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go
index 2655532..68de0e8 100644
--- a/internal/control/agentsvc/service.go
+++ b/internal/control/agentsvc/service.go
@@ -345,8 +345,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
problems := hb.GetNewProblems()
storage := hb.GetStorage()
memMB := int(hb.GetMemoryTotalMb())
+ cpus := int(hb.GetCpuCount())
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
- !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
+ !hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
return
@@ -406,6 +407,12 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
"from_mb", n.MemoryTotalMB, "to_mb", memMB)
n.MemoryTotalMB = memMB
}
+ if cpus > 0 && cpus != n.CPUCount {
+ if n.CPUCount > 0 {
+ s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus)
+ }
+ n.CPUCount = cpus
+ }
if hb.GetEnginesChanged() {
vers := maps.Clone(hb.GetEngineVersions())
if !maps.Equal(vers, n.EngineVersions) {
diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go
index b64277c..95ae8b0 100644
--- a/internal/control/catalog/catalog.go
+++ b/internal/control/catalog/catalog.go
@@ -74,6 +74,17 @@ type Profile struct {
// the control plane.
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
+ // CPUCores is the cores a replica of a profile with no GPU is given:
+ // reserved out of its node's CPUs by placement, as memory is, and what
+ // its engine computes on (llama.cpp's --threads).
+ //
+ // Reserved because an engine takes every core it is told about, and two
+ // on one node that each think they have all of them spin against each
+ // other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
+ // to under one each. Zero reserves nothing and leaves the thread count to
+ // the node, which is every profile from before this.
+ CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"`
+
// MinDiskGB is the free space a node needs before it can hold these
// weights.
//
@@ -380,6 +391,7 @@ func (m Model) PlacementProfiles() []placement.Profile {
NodeSelector: p.NodeSelector,
MinMemoryGB: p.MinMemoryGB,
MinDiskGB: p.MinDiskGB,
+ CPUCores: p.CPUCores,
// The model's source, so placement can prefer nodes that already
// hold the weights rather than one that must first move them.
Source: m.Source,
@@ -475,3 +487,19 @@ func (p Profile) Fingerprint() string {
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}
+
+// SuggestCores is the cores to give a replica with no GPU on nodes of these
+// CPU counts: half of the smallest, so two replicas share any of them without
+// competing. Zero when no node says, which reserves nothing.
+func SuggestCores(cpuCounts []int) int {
+ smallest := 0
+ for _, c := range cpuCounts {
+ if c > 0 && (smallest == 0 || c < smallest) {
+ smallest = c
+ }
+ }
+ if smallest == 0 {
+ return 0
+ }
+ return max(smallest/2, 1)
+}
diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go
index 8bfcce5..dc92159 100644
--- a/internal/control/inventory/inventory.go
+++ b/internal/control/inventory/inventory.go
@@ -79,6 +79,9 @@ type Node struct {
// it, which is treated as no constraint — the same as every node before
// this field existed.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
+ // CPUCount is the CPUs the agent may run on. Zero is an agent too old to
+ // say, which, like memory, constrains nothing.
+ CPUCount int `json:"cpu_count,omitempty"`
// Weights are the model files this node can see, however they got there.
// Empty means the node was not asked to look, which is not the same as
@@ -373,6 +376,9 @@ type Allocation struct {
// Zero for a profile that declared nothing, which constrains nothing and
// is counted as nothing.
MemoryGB int
+ // CPUCores is the cores this shard was given to compute on, for a shard
+ // with no GPU; zero for one that shares whatever is free.
+ CPUCores int
}
// Store holds the cluster inventory. It is safe for concurrent use.
diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go
index 68429e9..6d22a2d 100644
--- a/internal/control/modelserve/modelserve.go
+++ b/internal/control/modelserve/modelserve.go
@@ -91,6 +91,9 @@ type Request struct {
// use. They default to 16384 and 4096, what four parallel slots divide.
CtxPool int `json:"ctx_pool,omitempty"`
ContextLength int `json:"context_length,omitempty"`
+ // CPUCores is what a replica computes on; zero takes half the smallest
+ // node it fits on.
+ CPUCores int `json:"cpu_cores,omitempty"`
// By is who asked, for the catalog's record.
By string `json:"-"`
}
@@ -278,13 +281,20 @@ func (s *Server) run(j *Job) {
}
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
+ cores := req.CPUCores
+ if cores <= 0 {
+ if nc, ok := s.deps.Fleet.(interface{ NodeCPUs(int) []int }); ok {
+ cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
+ }
+ }
+
// 4. The catalog.
m := catalog.Model{
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
Profiles: []catalog.Profile{{
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
- Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB,
+ Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
}},
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
}
diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go
index 94c7621..dbdbf61 100644
--- a/internal/control/modelsvc/modelsvc.go
+++ b/internal/control/modelsvc/modelsvc.go
@@ -18,6 +18,7 @@ package modelsvc
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"log/slog"
"sync"
@@ -104,6 +105,16 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
if s.Declared(m.Name) {
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
}
+ // A model with no GPU is all host memory, and one that declares none
+ // fits everywhere as far as placement can tell — however many of it
+ // there already are. Required of what is added now; models stored before
+ // this still load, and say what they are missing.
+ for _, p := range m.Profiles {
+ if p.CPUOnly() && p.MinMemoryGB <= 0 {
+ return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+
+ "placement could not tell when a node is full", m.Name, p.Name)
+ }
+ }
if s.check != nil {
if err := s.check(ctx, m); err != nil {
return err
@@ -189,36 +200,72 @@ func (s *Service) Remove(ctx context.Context, name string) error {
return nil
}
-// Scale changes how many replicas a model runs, without a restart.
-//
-// Only the scaling block moves. Everything else about the model — its source,
-// its profiles, the engine arguments somebody tuned — is read back from what
-// was stored and written out again untouched, so changing a number cannot
-// quietly become a redefinition.
-//
-// A model declared in configuration is refused, for the same reason Remove
-// refuses one: the YAML is read on every start, so the new number would hold
-// until the next restart and then silently revert. A cluster that disagrees
-// with its own config file is worse than an error message saying where to edit.
+// Scale changes how many replicas a model added at runtime runs.
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
+ final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
+ m.Scaling = sc
+ return nil
+ })
+ if err == nil {
+ s.log.Info("model scaled", "model", name,
+ "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
+ }
+ return final, err
+}
+
+// Resources sets what a model's profile reserves of a node: host memory, and
+// the CPU cores a replica without a GPU computes on. An empty profile name
+// sets every profile.
+func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) {
+ if memoryGB < 0 || cpuCores < 0 {
+ return catalog.Model{}, errors.New("memory and cores cannot be negative")
+ }
+ final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error {
+ found := false
+ for i := range m.Profiles {
+ if profile != "" && m.Profiles[i].Name != profile {
+ continue
+ }
+ found = true
+ m.Profiles[i].MinMemoryGB = memoryGB
+ m.Profiles[i].CPUCores = cpuCores
+ }
+ if !found {
+ return fmt.Errorf("model %q has no profile %q", name, profile)
+ }
+ return nil
+ })
+ if err == nil {
+ s.log.Info("model resources set", "model", name, "profile", profile,
+ "memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by)
+ }
+ return final, err
+}
+
+// update changes one model added at runtime and keeps the change.
+//
+// Built before anything is written, like Add: catalog.New is what applies the
+// defaults and refuses what does not validate, so the model that comes back
+// is the one that will run rather than the one that was asked for.
+func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.Declared(name) {
return catalog.Model{}, fmt.Errorf(
- "model %q is declared in configuration; change its scaling there, or it reverts on the next restart", name)
+ "model %q is declared in configuration; %s there, or it reverts on the next restart", name, what)
}
-
stored, err := s.storedModels(ctx)
if err != nil {
return catalog.Model{}, err
}
-
next := append([]catalog.Model{}, s.declared...)
found := false
for _, e := range stored {
if e.Name == name {
- e.Scaling = sc
+ if err := change(&e); err != nil {
+ return catalog.Model{}, err
+ }
found = true
}
next = append(next, e)
@@ -226,20 +273,14 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
if !found {
return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
}
-
- // Built before anything is written, like Add: catalog.New is what applies
- // the defaults (max follows min when it is left out) and what refuses a
- // max below a min, so the model that comes back is the one that will run
- // rather than the one that was asked for.
built, err := catalog.New(next)
if err != nil {
- return catalog.Model{}, fmt.Errorf("scaling %q that way would make the catalog invalid: %w", name, err)
+ return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err)
}
final, ok := built.Resolve(name)
if !ok {
return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
}
-
def, err := json.Marshal(final)
if err != nil {
return catalog.Model{}, err
@@ -249,10 +290,7 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
}); err != nil {
return catalog.Model{}, err
}
-
s.apply.ReplaceCatalog(ctx, built)
- s.log.Info("model scaled", "model", name,
- "min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
return final, nil
}
diff --git a/internal/control/modelsvc/modelsvc_test.go b/internal/control/modelsvc/modelsvc_test.go
index 63b49dd..bd72e95 100644
--- a/internal/control/modelsvc/modelsvc_test.go
+++ b/internal/control/modelsvc/modelsvc_test.go
@@ -48,7 +48,7 @@ func model(name string) catalog.Model {
return catalog.Model{
Name: name, Source: "/models/" + name,
Scaling: catalog.Scaling{Min: 1, Max: 1},
- Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
+ Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1, MinMemoryGB: 2}},
}
}
@@ -337,3 +337,43 @@ func TestMinZeroWithAMaxIsOnDemand(t *testing.T) {
t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout)
}
}
+
+func TestACPUModelAddedNowMustSayHowMuchMemoryItNeeds(t *testing.T) {
+ s, st, _ := newService(t)
+ m := model("bare")
+ m.Profiles[0].MinMemoryGB = 0
+ err := s.Add(t.Context(), m, "tester")
+ if err == nil || !strings.Contains(err.Error(), "declares no memory") {
+ t.Fatalf("a CPU model with no memory was added: %v", err)
+ }
+ if len(st.rows) != 0 {
+ t.Error("it was stored anyway")
+ }
+}
+
+func TestResourcesAreSetOnAStoredModelAndSurviveARestart(t *testing.T) {
+ s, _, _ := newService(t)
+ if err := s.Add(t.Context(), model("m"), "tester"); err != nil {
+ t.Fatal(err)
+ }
+ got, err := s.Resources(t.Context(), "m", "", 6, 3, "tester")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if p := got.Profiles[0]; p.MinMemoryGB != 6 || p.CPUCores != 3 {
+ t.Errorf("profile %+v", p)
+ }
+ if _, err := s.Resources(t.Context(), "m", "nope", 6, 3, "tester"); err == nil {
+ t.Error("a profile the model does not have was accepted")
+ }
+ if _, err := s.Resources(t.Context(), "m", "", -1, 0, "tester"); err == nil {
+ t.Error("negative memory was accepted")
+ }
+ cat, err := s.Load(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if m, _ := cat.Resolve("m"); m.Profiles[0].CPUCores != 3 {
+ t.Error("the cores were lost on reload")
+ }
+}
diff --git a/internal/control/placement/cpu_test.go b/internal/control/placement/cpu_test.go
index d5c4891..5f72dc1 100644
--- a/internal/control/placement/cpu_test.go
+++ b/internal/control/placement/cpu_test.go
@@ -1,6 +1,7 @@
package placement
import (
+ "strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
@@ -100,3 +101,38 @@ func TestCPUProfileRespectsAvoid(t *testing.T) {
t.Errorf("placed on %q despite being told to avoid it", got)
}
}
+
+func TestCPUCoresAreReservedAsMemoryIs(t *testing.T) {
+ n := node("n1", "", 0)
+ n.CPUCount = 8
+ st := store(n)
+ s := New(st)
+ p := cpuProfile("cpu")
+ p.CPUCores = 3
+
+ for i := range 2 {
+ res := s.Place([]Profile{p}, nil)
+ if res.Placement == nil {
+ t.Fatalf("replica %d did not fit in 8 cores:\n%s", i+1, res.Explain())
+ }
+ st.Allocate("n1", inventory.Allocation{ShardID: string(rune('a' + i)), CPUCores: 3})
+ }
+ // Six of eight given away: a third would need three.
+ res := s.Place([]Profile{p}, nil)
+ if res.Placement != nil {
+ t.Fatal("a third replica was given cores the node does not have")
+ }
+ if got := res.Explain(); !strings.Contains(got, "has 8 CPUs with 2 free, needs 3") {
+ t.Errorf("the refusal should say how many cores are left:\n%s", got)
+ }
+
+ // Unknown counts constrain nothing: an older agent, or a profile that
+ // reserves no cores.
+ old := node("n2", "", 0)
+ if res := New(store(old)).Place([]Profile{p}, nil); res.Placement == nil {
+ t.Error("a node that does not report its CPUs was excluded")
+ }
+ if res := s.Place([]Profile{cpuProfile("cpu")}, nil); res.Placement == nil {
+ t.Error("a profile reserving no cores was refused")
+ }
+}
diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go
index 59ee579..fdb92d7 100644
--- a/internal/control/placement/placement.go
+++ b/internal/control/placement/placement.go
@@ -39,6 +39,9 @@ type Profile struct {
// MinDiskGB is the free space needed to hold the weights, checked only on
// nodes that do not already have them. Zero is unconstrained.
MinDiskGB int
+ // CPUCores is the cores a profile with no GPU reserves. Zero is
+ // unconstrained.
+ CPUCores int
// Source is the model's weights, as the engine would be given it. Used to
// prefer nodes that already hold them; empty asks for no preference.
Source string
@@ -582,6 +585,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
}
}
+ // Cores, for a replica that computes on them: what is left after the
+ // cores already given to the node's other replicas. Same exemptions
+ // as memory: zero declares nothing, and a node that does not report
+ // its CPUs is an older agent, not one with none.
+ if p.CPUOnly() && p.CPUCores > 0 && n.CPUCount > 0 {
+ free := n.CPUCount - s.claimedCores(n.ID)
+ if free < p.CPUCores {
+ reject(fmt.Sprintf("has %d CPUs with %d free, needs %d", n.CPUCount, free, p.CPUCores))
+ continue
+ }
+ }
+
if p.CPUOnly() {
// Every ready node can run it; there is no card to be free.
out = append(out, candidate{node: n})
@@ -723,6 +738,15 @@ func (s *Scheduler) claimedMemory(nodeID string) int {
return total
}
+// claimedCores is the CPU cores already given to replicas on a node.
+func (s *Scheduler) claimedCores(nodeID string) int {
+ total := 0
+ for _, a := range s.inv.Allocations(nodeID) {
+ total += a.CPUCores
+ }
+ return total
+}
+
// installing says why an engine is missing, when the node has said.
func installing(n inventory.Node, engine string) string {
for _, p := range n.Provision {
diff --git a/internal/control/reconciler/adopt.go b/internal/control/reconciler/adopt.go
index 8a27c38..b5830e3 100644
--- a/internal/control/reconciler/adopt.go
+++ b/internal/control/reconciler/adopt.go
@@ -144,6 +144,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
r.inv.Allocate(s.NodeID, inventory.Allocation{
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
+ CPUCores: r.adoptedCores(rep.Model, rep.Profile),
})
// The endpoint comes from the leader, which may be in any of the
// groups — the rank 0 node is not necessarily the first to register.
@@ -253,3 +254,9 @@ type AdoptedShard struct {
// Endpoint is the leader's base URL, set on the rank 0 shard.
Endpoint string
}
+
+// adoptedCores is what an adopted shard is taken to hold of its node's CPUs.
+func (r *Reconciler) adoptedCores(model, profile string) int {
+ p, _ := r.profileOf(model, profile)
+ return cpuCores(p)
+}
diff --git a/internal/control/reconciler/headroom.go b/internal/control/reconciler/headroom.go
index fae46a5..6080b03 100644
--- a/internal/control/reconciler/headroom.go
+++ b/internal/control/reconciler/headroom.go
@@ -66,14 +66,14 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
h.Blocked = slices.Compact(h.Blocked)
return h, true
}
- mem := 0
+ mem, cores := 0, 0
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
- mem = prof.MinMemoryGB
+ mem, cores = prof.MinMemoryGB, cpuCores(prof)
}
var nodes []string
for _, a := range res.Placement.Nodes {
inv.Allocate(a.NodeID, inventory.Allocation{
- ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem,
+ ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores,
})
nodes = append(nodes, a.NodeID)
}
diff --git a/internal/control/reconciler/headroom_test.go b/internal/control/reconciler/headroom_test.go
index 0bf3c1d..44f9111 100644
--- a/internal/control/reconciler/headroom_test.go
+++ b/internal/control/reconciler/headroom_test.go
@@ -57,3 +57,24 @@ func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) {
t.Errorf("headroom %+v, expected room for 3 more", got)
}
}
+
+func TestReservedCoresAreTheEnginesThreadsAndBoundTheRoom(t *testing.T) {
+ cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 64 * 1024, CPUCount: 8}
+ m := catalog.Model{Name: "coder", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
+ Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6, CPUCores: 3}}}
+ h := newHarness(t, m, cpu)
+ h.r.Reconcile(t.Context())
+
+ started := h.disp.startedShards()
+ if len(started) != 1 || started[0].Spec.Threads != 3 {
+ t.Fatalf("the engine was not told its 3 cores: %+v", started)
+ }
+ if a := h.inv.Allocations("a"); len(a) != 1 || a[0].CPUCores != 3 {
+ t.Errorf("the cores were not reserved: %+v", a)
+ }
+ // Memory would allow nine more; cores allow one: 8 - 3 = 5, then 2.
+ got, _ := h.r.Headroom("coder", 10)
+ if got.More != 1 {
+ t.Errorf("room for %d more, expected 1 by cores: %+v", got.More, got)
+ }
+}
diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go
index 9f86bd6..8572f08 100644
--- a/internal/control/reconciler/reconciler.go
+++ b/internal/control/reconciler/reconciler.go
@@ -825,7 +825,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
- MemoryGB: prof.MinMemoryGB,
+ MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof),
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
@@ -869,6 +869,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
HTTPPort: s.HTTPPort,
Profile: espec,
APIKey: r.engineKey(id),
+ Threads: cpuCores(prof),
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
@@ -1108,16 +1109,31 @@ func addrOf(n inventory.Node) string {
// claim on restart would place a second large model onto a node already full of
// the first.
func (r *Reconciler) profileMemory(model, profile string) int {
+ p, _ := r.profileOf(model, profile)
+ return p.MinMemoryGB
+}
+
+// profileOf is a replica's profile as the catalog holds it now.
+func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) {
m, ok := r.cat.Load().Resolve(model)
if !ok {
- return 0
+ return catalog.Profile{}, false
}
for _, p := range m.Profiles {
if p.Name == profile {
- return p.MinMemoryGB
+ return p, true
}
}
- return 0
+ return catalog.Profile{}, false
+}
+
+// cpuCores is what a profile reserves of a node's CPUs: its cores when it
+// computes on them, nothing when a GPU does the work.
+func cpuCores(p catalog.Profile) int {
+ if !p.CPUOnly() {
+ return 0
+ }
+ return p.CPUCores
}
func (r *Reconciler) engineKey(replicaID string) string {
diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go
index 98cfa1e..4ac7666 100644
--- a/internal/web/admin_serve.go
+++ b/internal/web/admin_serve.go
@@ -49,6 +49,9 @@ type serveForm struct {
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
// told what the model needed.
MinMemoryGB int
+ // CPUCores is the cores a replica computes on: half the smallest node it
+ // fits on, or zero when no node says how many it has.
+ CPUCores int
// MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
MemoryBasis string
// CtxPool is the engine context pool the form starts with, and the one
@@ -152,6 +155,15 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
defaultCtxPool, header.BlockCount, kvHeads,
float64(int64(1<<30)+weights/20)/(1<<30))
}
+ if fleet != nil {
+ var cpus []int
+ for _, n := range fleet.Nodes() {
+ if n.State == "ready" && (n.MemoryGB == 0 || n.MemoryGB >= f.MinMemoryGB) {
+ cpus = append(cpus, n.CPUCount)
+ }
+ }
+ f.CPUCores = catalog.SuggestCores(cpus)
+ }
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
f.Warning = strings.TrimSuffix(f.Warning, ".") +
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
@@ -277,6 +289,11 @@ func buildModel(get func(string) string) (catalog.Model, error) {
return m, err
}
}
+ if v := strings.TrimSpace(get("cpu_cores")); v != "" {
+ if p.CPUCores, err = positive(v, "the cores"); err != nil {
+ return m, err
+ }
+ }
if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" {
// What keeps a large model off the small nodes. A CPU profile has no
// card to be short of, so without this every ready node looks like a
diff --git a/internal/web/static/base.css b/internal/web/static/base.css
index b98f799..489bc2e 100644
--- a/internal/web/static/base.css
+++ b/internal/web/static/base.css
@@ -854,3 +854,4 @@ table.explore td:first-child { max-width: 34rem; overflow: hidden; text-overflow
table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted); }
.clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis;
vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; }
+table.rows td.warn { color: var(--warn); cursor: help; }
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index a97349c..42aba77 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -165,7 +165,9 @@ reachable from this machine.