Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
990d82d801 | ||
|
|
26151ec25a | ||
|
|
235bf33dcd |
No files matched your search
@@ -25,6 +25,30 @@ APIs change between minors without a deprecation period.
|
||||
Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until
|
||||
there are enough for v0.10.5.
|
||||
|
||||
### CPU cores are reserved, and memory has to be declared
|
||||
|
||||
The first step of resource allocation for larger clusters: what a replica
|
||||
holds of a node is reserved, not assumed.
|
||||
|
||||
- **Cores.** A profile's `cpu_cores` is reserved out of its node's CPUs, as
|
||||
memory is, and is what its engine computes on (llama.cpp's `--threads`). A
|
||||
node holds as many CPU replicas as it has cores for, and two on one node no
|
||||
longer spin against each other. Agents report their CPU count; a node that
|
||||
does not, and a profile that declares no cores, constrain nothing.
|
||||
- **Memory is required of new CPU models.** A model with no GPU is all host
|
||||
memory, and one declaring none looked to placement as if it fitted
|
||||
everywhere, any number of times. Adding one without `min_memory_gb` is now
|
||||
refused. Models stored before this still load, and say what is missing.
|
||||
- **Declared against measured.** Each model shows what one replica reserves
|
||||
and the most one of its replicas uses — in `ctl models` (RESERVES) and on the
|
||||
cluster page — with a warning when nothing is declared or when it uses more
|
||||
than a tenth over what it reserves.
|
||||
- **Setting them.** `llmgw ctl model resources MODEL --memory GB --cores N`
|
||||
(and `POST /api/v1/models/{name}/resources`) for a model added at runtime.
|
||||
`ctl model serve` and the mirror's serve form fill in cores — half the
|
||||
smallest node the model fits on — beside the memory they already estimate.
|
||||
Running replicas keep their old figures until restarted.
|
||||
|
||||
### The mirror page in tabs
|
||||
|
||||
/admin/mirror was one long page: what the mirror holds, the Hugging Face
|
||||
@@ -55,6 +79,19 @@ in amber when short, in red when parked or when it does not fit. It is one line
|
||||
per model beside its mode, and nodes drop the part of the name every node
|
||||
shares (node05 for llmgw-node05, the full name on hover).
|
||||
|
||||
### One line per record, in every table
|
||||
|
||||
The same rule across the admin pages. Headers do not wrap; a record is one
|
||||
line; free text is cut to its column with the whole of it on hover rather than
|
||||
given a row of its own; a row across the whole table is only a note or an open
|
||||
form. On the cluster page that means a node is one line (its address and logs
|
||||
beside its name, not under it), a model's description is a column rather than a
|
||||
second row, node names drop the prefix they all share, and the GPUs, classes,
|
||||
fabrics and fallbacks columns appear only when some row has something in them —
|
||||
on a CPU-only cluster they were three columns of dashes pushing the table past
|
||||
the page. On People, a person in many teams shows two and a count, the rest on
|
||||
hover.
|
||||
|
||||
## v0.10.4 — 2026-09-30
|
||||
|
||||
Published first as release candidates v0.10.4-rc.1 to rc.9.
|
||||
|
||||
@@ -534,6 +534,10 @@ type Register struct {
|
||||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||||
// built node with no engine installed yet was a candidate for every model.
|
||||
EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"`
|
||||
// cpu_count is the CPUs the agent may run on. Placement reserves cores
|
||||
// for CPU replicas out of it, the way it reserves memory out of
|
||||
// memory_total_mb; zero, from an older agent, reserves nothing.
|
||||
CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
|
||||
// imex is the service that lets a multi-node NVLink domain share memory.
|
||||
Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"`
|
||||
// installable_engines are the engines this agent knows how to install
|
||||
@@ -679,6 +683,13 @@ func (x *Register) GetEnginesProbed() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *Register) GetCpuCount() int32 {
|
||||
if x != nil {
|
||||
return x.CpuCount
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Register) GetImex() *IMEX {
|
||||
if x != nil {
|
||||
return x.Imex
|
||||
@@ -2017,8 +2028,10 @@ type Heartbeat struct {
|
||||
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
|
||||
// once a minute: a fabric manager restarting drops every card out of its
|
||||
// domain, and the node has to stop being offered work that spans it.
|
||||
GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
|
||||
Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
|
||||
GpuFabric []*GPUFabricState `protobuf:"bytes,16,rep,name=gpu_fabric,json=gpuFabric,proto3" json:"gpu_fabric,omitempty"`
|
||||
Imex *IMEX `protobuf:"bytes,17,opt,name=imex,proto3" json:"imex,omitempty"`
|
||||
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
||||
CpuCount int32 `protobuf:"varint,18,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -2172,6 +2185,13 @@ func (x *Heartbeat) GetImex() *IMEX {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetCpuCount() int32 {
|
||||
if x != nil {
|
||||
return x.CpuCount
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
// versions. Nothing more.
|
||||
//
|
||||
@@ -2561,7 +2581,10 @@ type StartShard struct {
|
||||
// api_key is what the engine demands of every request, derived by the
|
||||
// control plane from the replica id. The router sends it; nothing else
|
||||
// that can reach the node's ports has it.
|
||||
ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
|
||||
ApiKey string `protobuf:"bytes,16,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
|
||||
// cpu_threads is how many CPU threads a shard with no GPU computes on:
|
||||
// the cores placement reserved for it. Zero leaves it to the agent.
|
||||
CpuThreads int32 `protobuf:"varint,17,opt,name=cpu_threads,json=cpuThreads,proto3" json:"cpu_threads,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -2708,6 +2731,13 @@ func (x *StartShard) GetApiKey() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *StartShard) GetCpuThreads() int32 {
|
||||
if x != nil {
|
||||
return x.CpuThreads
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type EngineProfile struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
|
||||
@@ -3655,7 +3685,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\n" +
|
||||
" \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" +
|
||||
"\x03msg\"\xe3\x06\n" +
|
||||
"\x03msg\"\x80\a\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
|
||||
@@ -3671,7 +3701,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" +
|
||||
"\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" +
|
||||
"\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" +
|
||||
"\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" +
|
||||
"\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\x1b\n" +
|
||||
"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x12\"\n" +
|
||||
"\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" +
|
||||
"\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x12\x1a\n" +
|
||||
"\bfeatures\x18\x11 \x03(\tR\bfeatures\x1a9\n" +
|
||||
@@ -3811,7 +3842,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x06status\x18\x03 \x01(\tR\x06status\x12!\n" +
|
||||
"\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" +
|
||||
"\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" +
|
||||
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd9\x06\n" +
|
||||
"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xf6\x06\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
@@ -3831,7 +3862,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\afabrics\x18\x0f \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x127\n" +
|
||||
"\n" +
|
||||
"gpu_fabric\x18\x10 \x03(\v2\x18.llmgw.v1.GPUFabricStateR\tgpuFabric\x12\"\n" +
|
||||
"\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1aA\n" +
|
||||
"\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" +
|
||||
"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x1aA\n" +
|
||||
"\x13EngineVersionsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
|
||||
@@ -3872,7 +3904,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" +
|
||||
"\vbytes_total\x18\x03 \x01(\x03R\n" +
|
||||
"bytesTotal\"\xf2\x03\n" +
|
||||
"bytesTotal\"\x93\x04\n" +
|
||||
"\n" +
|
||||
"StartShard\x12\x1d\n" +
|
||||
"\n" +
|
||||
@@ -3896,7 +3928,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" +
|
||||
"\n" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
|
||||
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\"\xfd\x02\n" +
|
||||
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" +
|
||||
"\vcpu_threads\x18\x11 \x01(\x05R\n" +
|
||||
"cpuThreads\"\xfd\x02\n" +
|
||||
"\rEngineProfile\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
|
||||
|
||||
@@ -120,6 +120,10 @@ message Register {
|
||||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||||
// built node with no engine installed yet was a candidate for every model.
|
||||
bool engines_probed = 14;
|
||||
// cpu_count is the CPUs the agent may run on. Placement reserves cores
|
||||
// for CPU replicas out of it, the way it reserves memory out of
|
||||
// memory_total_mb; zero, from an older agent, reserves nothing.
|
||||
int32 cpu_count = 18;
|
||||
// imex is the service that lets a multi-node NVLink domain share memory.
|
||||
IMEX imex = 15;
|
||||
|
||||
@@ -420,6 +424,8 @@ message Heartbeat {
|
||||
// domain, and the node has to stop being offered work that spans it.
|
||||
repeated GPUFabricState gpu_fabric = 16;
|
||||
IMEX imex = 17;
|
||||
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
||||
int32 cpu_count = 18;
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
@@ -530,6 +536,9 @@ message StartShard {
|
||||
// control plane from the replica id. The router sends it; nothing else
|
||||
// that can reach the node's ports has it.
|
||||
string api_key = 16;
|
||||
// cpu_threads is how many CPU threads a shard with no GPU computes on:
|
||||
// the cores placement reserved for it. Zero leaves it to the agent.
|
||||
int32 cpu_threads = 17;
|
||||
}
|
||||
|
||||
message EngineProfile {
|
||||
|
||||
@@ -129,6 +129,7 @@ rather than repeated.
|
||||
| `args` | `[]string` | [Profile](#profile) |
|
||||
| `env` | `map[string]string` | [Profile](#profile) |
|
||||
| `min_memory_gb` | `int` | [Profile](#profile) |
|
||||
| `cpu_cores` | `int` | [Profile](#profile) |
|
||||
| `min_disk_gb` | `int` | [Profile](#profile) |
|
||||
| `node_selector` | `map[string]string` | [Profile](#profile) |
|
||||
| `weight` | `float64` | [Profile](#profile) |
|
||||
@@ -966,6 +967,20 @@ this. A node that does not report its memory is not excluded either —
|
||||
an older agent must not have its nodes quietly emptied by an upgrade of
|
||||
the control plane.
|
||||
|
||||
### `cpu_cores`
|
||||
|
||||
`int`
|
||||
|
||||
CPUCores is the cores a replica of a profile with no GPU is given:
|
||||
reserved out of its node's CPUs by placement, as memory is, and what
|
||||
its engine computes on (llama.cpp's --threads).
|
||||
|
||||
Reserved because an engine takes every core it is told about, and two
|
||||
on one node that each think they have all of them spin against each
|
||||
other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
||||
to under one each. Zero reserves nothing and leaves the thread count to
|
||||
the node, which is every profile from before this.
|
||||
|
||||
### `min_disk_gb`
|
||||
|
||||
`int`
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -549,6 +550,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
// and a node that grew from 7 GB to 32 GB has to be seen to have grown
|
||||
// without anybody restarting anything.
|
||||
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
|
||||
hb.CpuCount = int32(runtime.NumCPU())
|
||||
|
||||
a.reprobeEngines()
|
||||
a.mu.Lock()
|
||||
|
||||
@@ -76,6 +76,10 @@ type ShardSpec struct {
|
||||
// and accounting. Given to the engine in its environment rather than on
|
||||
// its command line, which every user on the node can read.
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
// Threads is the CPU cores placement reserved for a shard with no GPU,
|
||||
// and so what its engine computes on. Zero leaves it to the engine's own
|
||||
// configuration.
|
||||
Threads int `json:"threads,omitempty"`
|
||||
|
||||
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
||||
// are carried so the agent can report them back after a restart; Fabric
|
||||
|
||||
@@ -133,7 +133,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
args = append(args, "--n-gpu-layers", "999")
|
||||
}
|
||||
if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
|
||||
n := l.cfg.Threads
|
||||
// What placement reserved first: it is what the node's other
|
||||
// replicas were placed around.
|
||||
n := s.Threads
|
||||
if n <= 0 {
|
||||
n = l.cfg.Threads
|
||||
}
|
||||
if n <= 0 {
|
||||
n = max(numCPU()/2, 1)
|
||||
}
|
||||
|
||||
@@ -294,3 +294,15 @@ func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
|
||||
t.Errorf("--threads %q on one CPU", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
|
||||
s := llamaSpec(nil)
|
||||
s.Threads = 3
|
||||
cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
|
||||
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
|
||||
}
|
||||
}
|
||||
+34
-4
@@ -139,6 +139,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newStatusCmd(client, &asJSON),
|
||||
newModelCmd(client),
|
||||
newScaleCmd(client),
|
||||
newResourcesCmd(client),
|
||||
newRestartCmd(client),
|
||||
newRetryCmd(client),
|
||||
newUpgradeCmd(client),
|
||||
@@ -323,17 +324,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS")
|
||||
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
|
||||
var warnings []string
|
||||
for _, m := range models {
|
||||
serving := "no"
|
||||
if m.Ready > 0 {
|
||||
serving = "yes"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n",
|
||||
m.Name, serving, m.Ready, m.Min, modelStatus(m),
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\n",
|
||||
m.Name, serving, m.Ready, m.Min, modelStatus(m), reserves(m),
|
||||
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
|
||||
for _, x := range m.Warnings {
|
||||
warnings = append(warnings, m.Name+": "+x)
|
||||
}
|
||||
}
|
||||
return w.Flush()
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, x := range warnings {
|
||||
fmt.Println("warning:", x)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -511,3 +522,22 @@ func dash(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// reserves is what one replica of a model holds of a node, with what it was
|
||||
// measured using beside it: "6G 2c (5.1G used)".
|
||||
func reserves(m adminapi.ModelView) string {
|
||||
if m.MemoryGB == 0 && m.CPUCores == 0 {
|
||||
return "-"
|
||||
}
|
||||
out := fmt.Sprintf("%dG", m.MemoryGB)
|
||||
if m.CPUCores > 0 {
|
||||
out += fmt.Sprintf(" %dc", m.CPUCores)
|
||||
}
|
||||
if m.MeasuredMemoryGB > 0 {
|
||||
out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB)
|
||||
}
|
||||
if len(m.Warnings) > 0 {
|
||||
out += " !"
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -192,3 +192,44 @@ func newScaleCmd(client func() *ctlClient) *cobra.Command {
|
||||
"how long an on-demand model stays up unasked-for; zero takes the cluster default")
|
||||
return c
|
||||
}
|
||||
|
||||
func newResourcesCmd(client func() *ctlClient) *cobra.Command {
|
||||
var memory, cores int
|
||||
var profile string
|
||||
c := &cobra.Command{
|
||||
Use: "resources <model> --memory GB [--cores N]",
|
||||
Short: "Set what a model reserves of a node: memory, and CPU cores",
|
||||
Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" +
|
||||
"weights, KV cache and engine together. Placement reserves it on the node,\n" +
|
||||
"so a node holds only as many replicas as it has memory for.\n\n" +
|
||||
"--cores is the CPU cores a replica with no GPU computes on. They are\n" +
|
||||
"reserved the same way, and become the engine's thread count, so two\n" +
|
||||
"replicas on one node stop competing for the same cores. Zero reserves\n" +
|
||||
"none and leaves the thread count to the node.\n\n" +
|
||||
"Running replicas keep what they were started with; `llmgw ctl restart\n" +
|
||||
"--model M` moves them to the new figures.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if !cmd.Flags().Changed("memory") {
|
||||
return fmt.Errorf("--memory is required")
|
||||
}
|
||||
body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var m catalog.Model
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB")
|
||||
c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none")
|
||||
c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
|
||||
return c
|
||||
}
|
||||
@@ -75,6 +75,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
||||
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
||||
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
||||
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
|
||||
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
||||
return c
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
func TestAClaimedActorIsOneShortName(t *testing.T) {
|
||||
@@ -22,3 +24,22 @@ func TestAClaimedActorIsOneShortName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReservationThatPlacementCannotTrustIsSaid(t *testing.T) {
|
||||
cpu := func(mem int) catalog.Model {
|
||||
return catalog.Model{Name: "m", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: mem}}}
|
||||
}
|
||||
if w := reservationWarnings(cpu(0), 0); len(w) != 1 || !strings.Contains(w[0], "declares no memory") {
|
||||
t.Errorf("no memory declared: %v", w)
|
||||
}
|
||||
if w := reservationWarnings(cpu(6), 8.2); len(w) != 1 || !strings.Contains(w[0], "uses 8.2 GB and reserves 6") {
|
||||
t.Errorf("under-declared: %v", w)
|
||||
}
|
||||
if w := reservationWarnings(cpu(6), 6.4); len(w) != 0 {
|
||||
t.Errorf("within a tenth is mapped weights, not a wrong figure: %v", w)
|
||||
}
|
||||
gpu := catalog.Model{Name: "g", Profiles: []catalog.Profile{{Name: "tp1", Engine: "vllm", GPUClass: "h100-80g"}}}
|
||||
if w := reservationWarnings(gpu, 3); len(w) != 0 {
|
||||
t.Errorf("a GPU model needs no host memory figure: %v", w)
|
||||
}
|
||||
}
|
||||
@@ -57,6 +57,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models", a.addModel)
|
||||
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
|
||||
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
|
||||
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
|
||||
@@ -119,6 +120,8 @@ type NodeView struct {
|
||||
// what decides whether a CPU profile can be placed here, so it belongs
|
||||
// where someone looks when a model will not schedule.
|
||||
MemoryGB int `json:"memory_gb,omitempty"`
|
||||
// CPUCount is the machine's CPUs, 0 when the agent did not report them.
|
||||
CPUCount int `json:"cpu_count,omitempty"`
|
||||
// Engines are the inference engines installed here, as the agent found
|
||||
// them. Absent means the agent is older than the probe and said nothing,
|
||||
// which is not the same as having none — placement treats it that way too,
|
||||
@@ -270,6 +273,15 @@ type ModelView struct {
|
||||
Unplaceable bool `json:"unplaceable,omitempty"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
Profiles []string `json:"profiles"`
|
||||
// MemoryGB and CPUCores are what one replica reserves, from its first
|
||||
// profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
|
||||
// when none is measured.
|
||||
MemoryGB int `json:"memory_gb,omitempty"`
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"`
|
||||
// Warnings are what makes its reservation wrong: nothing declared where
|
||||
// placement needs a figure, or less declared than it uses.
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
@@ -396,8 +408,8 @@ func (a *API) Nodes() []NodeView {
|
||||
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
|
||||
State: n.State.String(), Labels: n.Labels,
|
||||
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
|
||||
MemoryGB: n.MemoryTotalMB / 1024,
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||
MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount,
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||
Replicas: replicasOn[n.ID],
|
||||
}
|
||||
if free, total := n.LocalDisk(); total > 0 {
|
||||
@@ -497,6 +509,15 @@ func (a *API) Models() []ModelView {
|
||||
for _, h := range a.ctl.Reconciler().ModelHealth() {
|
||||
health[h.Model] = h
|
||||
}
|
||||
// The most one replica of each model uses, summed over its shards.
|
||||
measured := map[string]int64{}
|
||||
for _, rep := range a.Replicas("") {
|
||||
var b int64
|
||||
for _, sh := range rep.Shards {
|
||||
b += sh.MemoryBytes
|
||||
}
|
||||
measured[rep.Model] = max(measured[rep.Model], b)
|
||||
}
|
||||
out := make([]ModelView, 0)
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
h := health[m.Name]
|
||||
@@ -511,6 +532,11 @@ func (a *API) Models() []ModelView {
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
}
|
||||
if len(m.Profiles) > 0 {
|
||||
v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
|
||||
}
|
||||
v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
|
||||
v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
@@ -581,6 +607,15 @@ type Models interface {
|
||||
Add(ctx context.Context, m catalog.Model, by string) error
|
||||
Remove(ctx context.Context, name string) error
|
||||
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
type ResourcesRequest struct {
|
||||
// Profile is one profile's name; empty is every profile.
|
||||
Profile string `json:"profile,omitempty"`
|
||||
MemoryGB int `json:"memory_gb"`
|
||||
CPUCores int `json:"cpu_cores"`
|
||||
}
|
||||
|
||||
// Usage reports what each group of the tenancy tree spent. The store
|
||||
@@ -826,6 +861,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// setResources sets a model's reserved memory and cores. Placed replicas keep
|
||||
// what they were started with until they are restarted.
|
||||
func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req ResourcesRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, m)
|
||||
}
|
||||
|
||||
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
@@ -1307,3 +1362,40 @@ func actorOf(r *http.Request) string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// NodeCPUs is the CPU counts of ready nodes with at least this much memory:
|
||||
// where a replica needing it could go.
|
||||
func (a *API) NodeCPUs(minMemoryGB int) []int {
|
||||
var out []int
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
if n.State.String() != "ready" || n.CPUCount == 0 {
|
||||
continue
|
||||
}
|
||||
if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB {
|
||||
continue
|
||||
}
|
||||
out = append(out, n.CPUCount)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reservationWarnings says what is wrong with what a model reserves.
|
||||
//
|
||||
// Resident memory includes the weights mapped from disk, which the kernel can
|
||||
// take back, so it overstates what a model needs by the part of the weights
|
||||
// not in use. A tenth over is let pass for that; more is a figure too low for
|
||||
// placement to be trusted with.
|
||||
func reservationWarnings(m catalog.Model, measuredGB float64) []string {
|
||||
var out []string
|
||||
for _, p := range m.Profiles {
|
||||
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
||||
out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+
|
||||
"(llmgw ctl model resources %s --memory GB)", p.Name, m.Name))
|
||||
}
|
||||
}
|
||||
if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 {
|
||||
out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory",
|
||||
measuredGB, m.Profiles[0].MinMemoryGB))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -258,6 +258,13 @@ type fakeModels struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeModels) Resources(_ context.Context, name, profile string, mem, cores int, _ string) (catalog.Model, error) {
|
||||
if f.err != nil {
|
||||
return catalog.Model{}, f.err
|
||||
}
|
||||
return catalog.Model{Name: name, Profiles: []catalog.Profile{{Name: "p", MinMemoryGB: mem, CPUCores: cores}}}, nil
|
||||
}
|
||||
|
||||
func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) {
|
||||
if f.err != nil {
|
||||
return catalog.Model{}, f.err
|
||||
|
||||
@@ -4,6 +4,7 @@ package agentsvc
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -27,6 +28,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
EngineVersions: r.GetEngineVersions(),
|
||||
EnginesProbed: r.GetEnginesProbed(),
|
||||
MemoryTotalMB: int(r.GetMemoryTotalMb()),
|
||||
CPUCount: int(r.GetCpuCount()),
|
||||
}
|
||||
if h := r.GetHost(); h != nil {
|
||||
n.Host = discovery.Host{
|
||||
@@ -90,6 +92,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr,
|
||||
Labels: labels, AgentVersion: agentVersion,
|
||||
MemoryTotalMb: int64(inv.MemoryTotalMB),
|
||||
CpuCount: int32(runtime.NumCPU()),
|
||||
}
|
||||
for _, w := range inv.Weights {
|
||||
r.Weights = append(r.Weights, &llmgwv1.LocalWeights{
|
||||
@@ -192,6 +195,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
Fabric: cmd.Fabric,
|
||||
ProfileFp: cmd.ProfileFP,
|
||||
ApiKey: cmd.Spec.APIKey,
|
||||
CpuThreads: int32(cmd.Spec.Threads),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -213,6 +217,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
Fabric: m.GetFabric(),
|
||||
ProfileFP: m.GetProfileFp(),
|
||||
APIKey: m.GetApiKey(),
|
||||
Threads: int(m.GetCpuThreads()),
|
||||
Profile: engines.Profile{
|
||||
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
|
||||
TP: int(p.GetTp()), PP: int(p.GetPp()),
|
||||
|
||||
@@ -345,8 +345,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
problems := hb.GetNewProblems()
|
||||
storage := hb.GetStorage()
|
||||
memMB := int(hb.GetMemoryTotalMb())
|
||||
cpus := int(hb.GetCpuCount())
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
||||
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
||||
return
|
||||
@@ -406,6 +407,12 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
"from_mb", n.MemoryTotalMB, "to_mb", memMB)
|
||||
n.MemoryTotalMB = memMB
|
||||
}
|
||||
if cpus > 0 && cpus != n.CPUCount {
|
||||
if n.CPUCount > 0 {
|
||||
s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus)
|
||||
}
|
||||
n.CPUCount = cpus
|
||||
}
|
||||
if hb.GetEnginesChanged() {
|
||||
vers := maps.Clone(hb.GetEngineVersions())
|
||||
if !maps.Equal(vers, n.EngineVersions) {
|
||||
|
||||
@@ -74,6 +74,17 @@ type Profile struct {
|
||||
// the control plane.
|
||||
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
||||
|
||||
// CPUCores is the cores a replica of a profile with no GPU is given:
|
||||
// reserved out of its node's CPUs by placement, as memory is, and what
|
||||
// its engine computes on (llama.cpp's --threads).
|
||||
//
|
||||
// Reserved because an engine takes every core it is told about, and two
|
||||
// on one node that each think they have all of them spin against each
|
||||
// other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
||||
// to under one each. Zero reserves nothing and leaves the thread count to
|
||||
// the node, which is every profile from before this.
|
||||
CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"`
|
||||
|
||||
// MinDiskGB is the free space a node needs before it can hold these
|
||||
// weights.
|
||||
//
|
||||
@@ -380,6 +391,7 @@ func (m Model) PlacementProfiles() []placement.Profile {
|
||||
NodeSelector: p.NodeSelector,
|
||||
MinMemoryGB: p.MinMemoryGB,
|
||||
MinDiskGB: p.MinDiskGB,
|
||||
CPUCores: p.CPUCores,
|
||||
// The model's source, so placement can prefer nodes that already
|
||||
// hold the weights rather than one that must first move them.
|
||||
Source: m.Source,
|
||||
@@ -475,3 +487,19 @@ func (p Profile) Fingerprint() string {
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// SuggestCores is the cores to give a replica with no GPU on nodes of these
|
||||
// CPU counts: half of the smallest, so two replicas share any of them without
|
||||
// competing. Zero when no node says, which reserves nothing.
|
||||
func SuggestCores(cpuCounts []int) int {
|
||||
smallest := 0
|
||||
for _, c := range cpuCounts {
|
||||
if c > 0 && (smallest == 0 || c < smallest) {
|
||||
smallest = c
|
||||
}
|
||||
}
|
||||
if smallest == 0 {
|
||||
return 0
|
||||
}
|
||||
return max(smallest/2, 1)
|
||||
}
|
||||
@@ -79,6 +79,9 @@ type Node struct {
|
||||
// it, which is treated as no constraint — the same as every node before
|
||||
// this field existed.
|
||||
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||
// CPUCount is the CPUs the agent may run on. Zero is an agent too old to
|
||||
// say, which, like memory, constrains nothing.
|
||||
CPUCount int `json:"cpu_count,omitempty"`
|
||||
|
||||
// Weights are the model files this node can see, however they got there.
|
||||
// Empty means the node was not asked to look, which is not the same as
|
||||
@@ -373,6 +376,9 @@ type Allocation struct {
|
||||
// Zero for a profile that declared nothing, which constrains nothing and
|
||||
// is counted as nothing.
|
||||
MemoryGB int
|
||||
// CPUCores is the cores this shard was given to compute on, for a shard
|
||||
// with no GPU; zero for one that shares whatever is free.
|
||||
CPUCores int
|
||||
}
|
||||
|
||||
// Store holds the cluster inventory. It is safe for concurrent use.
|
||||
|
||||
@@ -91,6 +91,9 @@ type Request struct {
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide.
|
||||
CtxPool int `json:"ctx_pool,omitempty"`
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||
// node it fits on.
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
// By is who asked, for the catalog's record.
|
||||
By string `json:"-"`
|
||||
}
|
||||
@@ -278,13 +281,20 @@ func (s *Server) run(j *Job) {
|
||||
}
|
||||
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
|
||||
|
||||
cores := req.CPUCores
|
||||
if cores <= 0 {
|
||||
if nc, ok := s.deps.Fleet.(interface{ NodeCPUs(int) []int }); ok {
|
||||
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
|
||||
}
|
||||
}
|
||||
|
||||
// 4. The catalog.
|
||||
m := catalog.Model{
|
||||
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||
Profiles: []catalog.Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||
}},
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ package modelsvc
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
@@ -104,6 +105,16 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
||||
if s.Declared(m.Name) {
|
||||
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
|
||||
}
|
||||
// A model with no GPU is all host memory, and one that declares none
|
||||
// fits everywhere as far as placement can tell — however many of it
|
||||
// there already are. Required of what is added now; models stored before
|
||||
// this still load, and say what they are missing.
|
||||
for _, p := range m.Profiles {
|
||||
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
||||
return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+
|
||||
"placement could not tell when a node is full", m.Name, p.Name)
|
||||
}
|
||||
}
|
||||
if s.check != nil {
|
||||
if err := s.check(ctx, m); err != nil {
|
||||
return err
|
||||
@@ -189,36 +200,72 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Scale changes how many replicas a model runs, without a restart.
|
||||
//
|
||||
// Only the scaling block moves. Everything else about the model — its source,
|
||||
// its profiles, the engine arguments somebody tuned — is read back from what
|
||||
// was stored and written out again untouched, so changing a number cannot
|
||||
// quietly become a redefinition.
|
||||
//
|
||||
// A model declared in configuration is refused, for the same reason Remove
|
||||
// refuses one: the YAML is read on every start, so the new number would hold
|
||||
// until the next restart and then silently revert. A cluster that disagrees
|
||||
// with its own config file is worse than an error message saying where to edit.
|
||||
// Scale changes how many replicas a model added at runtime runs.
|
||||
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
|
||||
m.Scaling = sc
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("model scaled", "model", name,
|
||||
"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// Resources sets what a model's profile reserves of a node: host memory, and
|
||||
// the CPU cores a replica without a GPU computes on. An empty profile name
|
||||
// sets every profile.
|
||||
func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) {
|
||||
if memoryGB < 0 || cpuCores < 0 {
|
||||
return catalog.Model{}, errors.New("memory and cores cannot be negative")
|
||||
}
|
||||
final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error {
|
||||
found := false
|
||||
for i := range m.Profiles {
|
||||
if profile != "" && m.Profiles[i].Name != profile {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
m.Profiles[i].MinMemoryGB = memoryGB
|
||||
m.Profiles[i].CPUCores = cpuCores
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("model %q has no profile %q", name, profile)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("model resources set", "model", name, "profile", profile,
|
||||
"memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// update changes one model added at runtime and keeps the change.
|
||||
//
|
||||
// Built before anything is written, like Add: catalog.New is what applies the
|
||||
// defaults and refuses what does not validate, so the model that comes back
|
||||
// is the one that will run rather than the one that was asked for.
|
||||
func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.Declared(name) {
|
||||
return catalog.Model{}, fmt.Errorf(
|
||||
"model %q is declared in configuration; change its scaling there, or it reverts on the next restart", name)
|
||||
"model %q is declared in configuration; %s there, or it reverts on the next restart", name, what)
|
||||
}
|
||||
|
||||
stored, err := s.storedModels(ctx)
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
e.Scaling = sc
|
||||
if err := change(&e); err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
found = true
|
||||
}
|
||||
next = append(next, e)
|
||||
@@ -226,20 +273,14 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
|
||||
if !found {
|
||||
return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
|
||||
}
|
||||
|
||||
// Built before anything is written, like Add: catalog.New is what applies
|
||||
// the defaults (max follows min when it is left out) and what refuses a
|
||||
// max below a min, so the model that comes back is the one that will run
|
||||
// rather than the one that was asked for.
|
||||
built, err := catalog.New(next)
|
||||
if err != nil {
|
||||
return catalog.Model{}, fmt.Errorf("scaling %q that way would make the catalog invalid: %w", name, err)
|
||||
return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err)
|
||||
}
|
||||
final, ok := built.Resolve(name)
|
||||
if !ok {
|
||||
return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
|
||||
}
|
||||
|
||||
def, err := json.Marshal(final)
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
@@ -249,10 +290,7 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
|
||||
}); err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
|
||||
s.apply.ReplaceCatalog(ctx, built)
|
||||
s.log.Info("model scaled", "model", name,
|
||||
"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
||||
return final, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ func model(name string) catalog.Model {
|
||||
return catalog.Model{
|
||||
Name: name, Source: "/models/" + name,
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
|
||||
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1, MinMemoryGB: 2}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -337,3 +337,43 @@ func TestMinZeroWithAMaxIsOnDemand(t *testing.T) {
|
||||
t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPUModelAddedNowMustSayHowMuchMemoryItNeeds(t *testing.T) {
|
||||
s, st, _ := newService(t)
|
||||
m := model("bare")
|
||||
m.Profiles[0].MinMemoryGB = 0
|
||||
err := s.Add(t.Context(), m, "tester")
|
||||
if err == nil || !strings.Contains(err.Error(), "declares no memory") {
|
||||
t.Fatalf("a CPU model with no memory was added: %v", err)
|
||||
}
|
||||
if len(st.rows) != 0 {
|
||||
t.Error("it was stored anyway")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourcesAreSetOnAStoredModelAndSurviveARestart(t *testing.T) {
|
||||
s, _, _ := newService(t)
|
||||
if err := s.Add(t.Context(), model("m"), "tester"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.Resources(t.Context(), "m", "", 6, 3, "tester")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p := got.Profiles[0]; p.MinMemoryGB != 6 || p.CPUCores != 3 {
|
||||
t.Errorf("profile %+v", p)
|
||||
}
|
||||
if _, err := s.Resources(t.Context(), "m", "nope", 6, 3, "tester"); err == nil {
|
||||
t.Error("a profile the model does not have was accepted")
|
||||
}
|
||||
if _, err := s.Resources(t.Context(), "m", "", -1, 0, "tester"); err == nil {
|
||||
t.Error("negative memory was accepted")
|
||||
}
|
||||
cat, err := s.Load(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := cat.Resolve("m"); m.Profiles[0].CPUCores != 3 {
|
||||
t.Error("the cores were lost on reload")
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
@@ -100,3 +101,38 @@ func TestCPUProfileRespectsAvoid(t *testing.T) {
|
||||
t.Errorf("placed on %q despite being told to avoid it", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUCoresAreReservedAsMemoryIs(t *testing.T) {
|
||||
n := node("n1", "", 0)
|
||||
n.CPUCount = 8
|
||||
st := store(n)
|
||||
s := New(st)
|
||||
p := cpuProfile("cpu")
|
||||
p.CPUCores = 3
|
||||
|
||||
for i := range 2 {
|
||||
res := s.Place([]Profile{p}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("replica %d did not fit in 8 cores:\n%s", i+1, res.Explain())
|
||||
}
|
||||
st.Allocate("n1", inventory.Allocation{ShardID: string(rune('a' + i)), CPUCores: 3})
|
||||
}
|
||||
// Six of eight given away: a third would need three.
|
||||
res := s.Place([]Profile{p}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatal("a third replica was given cores the node does not have")
|
||||
}
|
||||
if got := res.Explain(); !strings.Contains(got, "has 8 CPUs with 2 free, needs 3") {
|
||||
t.Errorf("the refusal should say how many cores are left:\n%s", got)
|
||||
}
|
||||
|
||||
// Unknown counts constrain nothing: an older agent, or a profile that
|
||||
// reserves no cores.
|
||||
old := node("n2", "", 0)
|
||||
if res := New(store(old)).Place([]Profile{p}, nil); res.Placement == nil {
|
||||
t.Error("a node that does not report its CPUs was excluded")
|
||||
}
|
||||
if res := s.Place([]Profile{cpuProfile("cpu")}, nil); res.Placement == nil {
|
||||
t.Error("a profile reserving no cores was refused")
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,9 @@ type Profile struct {
|
||||
// MinDiskGB is the free space needed to hold the weights, checked only on
|
||||
// nodes that do not already have them. Zero is unconstrained.
|
||||
MinDiskGB int
|
||||
// CPUCores is the cores a profile with no GPU reserves. Zero is
|
||||
// unconstrained.
|
||||
CPUCores int
|
||||
// Source is the model's weights, as the engine would be given it. Used to
|
||||
// prefer nodes that already hold them; empty asks for no preference.
|
||||
Source string
|
||||
@@ -582,6 +585,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
}
|
||||
}
|
||||
|
||||
// Cores, for a replica that computes on them: what is left after the
|
||||
// cores already given to the node's other replicas. Same exemptions
|
||||
// as memory: zero declares nothing, and a node that does not report
|
||||
// its CPUs is an older agent, not one with none.
|
||||
if p.CPUOnly() && p.CPUCores > 0 && n.CPUCount > 0 {
|
||||
free := n.CPUCount - s.claimedCores(n.ID)
|
||||
if free < p.CPUCores {
|
||||
reject(fmt.Sprintf("has %d CPUs with %d free, needs %d", n.CPUCount, free, p.CPUCores))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if p.CPUOnly() {
|
||||
// Every ready node can run it; there is no card to be free.
|
||||
out = append(out, candidate{node: n})
|
||||
@@ -723,6 +738,15 @@ func (s *Scheduler) claimedMemory(nodeID string) int {
|
||||
return total
|
||||
}
|
||||
|
||||
// claimedCores is the CPU cores already given to replicas on a node.
|
||||
func (s *Scheduler) claimedCores(nodeID string) int {
|
||||
total := 0
|
||||
for _, a := range s.inv.Allocations(nodeID) {
|
||||
total += a.CPUCores
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// installing says why an engine is missing, when the node has said.
|
||||
func installing(n inventory.Node, engine string) string {
|
||||
for _, p := range n.Provision {
|
||||
|
||||
@@ -144,6 +144,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
||||
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
|
||||
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
|
||||
CPUCores: r.adoptedCores(rep.Model, rep.Profile),
|
||||
})
|
||||
// The endpoint comes from the leader, which may be in any of the
|
||||
// groups — the rank 0 node is not necessarily the first to register.
|
||||
@@ -253,3 +254,9 @@ type AdoptedShard struct {
|
||||
// Endpoint is the leader's base URL, set on the rank 0 shard.
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
// adoptedCores is what an adopted shard is taken to hold of its node's CPUs.
|
||||
func (r *Reconciler) adoptedCores(model, profile string) int {
|
||||
p, _ := r.profileOf(model, profile)
|
||||
return cpuCores(p)
|
||||
}
|
||||
@@ -66,14 +66,14 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||
h.Blocked = slices.Compact(h.Blocked)
|
||||
return h, true
|
||||
}
|
||||
mem := 0
|
||||
mem, cores := 0, 0
|
||||
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
|
||||
mem = prof.MinMemoryGB
|
||||
mem, cores = prof.MinMemoryGB, cpuCores(prof)
|
||||
}
|
||||
var nodes []string
|
||||
for _, a := range res.Placement.Nodes {
|
||||
inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem,
|
||||
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores,
|
||||
})
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
|
||||
@@ -57,3 +57,24 @@ func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) {
|
||||
t.Errorf("headroom %+v, expected room for 3 more", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReservedCoresAreTheEnginesThreadsAndBoundTheRoom(t *testing.T) {
|
||||
cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 64 * 1024, CPUCount: 8}
|
||||
m := catalog.Model{Name: "coder", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6, CPUCores: 3}}}
|
||||
h := newHarness(t, m, cpu)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
started := h.disp.startedShards()
|
||||
if len(started) != 1 || started[0].Spec.Threads != 3 {
|
||||
t.Fatalf("the engine was not told its 3 cores: %+v", started)
|
||||
}
|
||||
if a := h.inv.Allocations("a"); len(a) != 1 || a[0].CPUCores != 3 {
|
||||
t.Errorf("the cores were not reserved: %+v", a)
|
||||
}
|
||||
// Memory would allow nine more; cores allow one: 8 - 3 = 5, then 2.
|
||||
got, _ := h.r.Headroom("coder", 10)
|
||||
if got.More != 1 {
|
||||
t.Errorf("room for %d more, expected 1 by cores: %+v", got.More, got)
|
||||
}
|
||||
}
|
||||
@@ -825,7 +825,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
})
|
||||
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
||||
MemoryGB: prof.MinMemoryGB,
|
||||
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof),
|
||||
})
|
||||
}
|
||||
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
||||
@@ -869,6 +869,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
HTTPPort: s.HTTPPort,
|
||||
Profile: espec,
|
||||
APIKey: r.engineKey(id),
|
||||
Threads: cpuCores(prof),
|
||||
},
|
||||
StartupTimeout: prof.StartupTimeout,
|
||||
Pool: pl.Pool,
|
||||
@@ -1108,16 +1109,31 @@ func addrOf(n inventory.Node) string {
|
||||
// claim on restart would place a second large model onto a node already full of
|
||||
// the first.
|
||||
func (r *Reconciler) profileMemory(model, profile string) int {
|
||||
p, _ := r.profileOf(model, profile)
|
||||
return p.MinMemoryGB
|
||||
}
|
||||
|
||||
// profileOf is a replica's profile as the catalog holds it now.
|
||||
func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
if !ok {
|
||||
return 0
|
||||
return catalog.Profile{}, false
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
if p.Name == profile {
|
||||
return p.MinMemoryGB
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return 0
|
||||
return catalog.Profile{}, false
|
||||
}
|
||||
|
||||
// cpuCores is what a profile reserves of a node's CPUs: its cores when it
|
||||
// computes on them, nothing when a GPU does the work.
|
||||
func cpuCores(p catalog.Profile) int {
|
||||
if !p.CPUOnly() {
|
||||
return 0
|
||||
}
|
||||
return p.CPUCores
|
||||
}
|
||||
|
||||
func (r *Reconciler) engineKey(replicaID string) string {
|
||||
|
||||
@@ -49,6 +49,9 @@ type serveForm struct {
|
||||
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
|
||||
// told what the model needed.
|
||||
MinMemoryGB int
|
||||
// CPUCores is the cores a replica computes on: half the smallest node it
|
||||
// fits on, or zero when no node says how many it has.
|
||||
CPUCores int
|
||||
// MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
|
||||
MemoryBasis string
|
||||
// CtxPool is the engine context pool the form starts with, and the one
|
||||
@@ -152,6 +155,15 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
defaultCtxPool, header.BlockCount, kvHeads,
|
||||
float64(int64(1<<30)+weights/20)/(1<<30))
|
||||
}
|
||||
if fleet != nil {
|
||||
var cpus []int
|
||||
for _, n := range fleet.Nodes() {
|
||||
if n.State == "ready" && (n.MemoryGB == 0 || n.MemoryGB >= f.MinMemoryGB) {
|
||||
cpus = append(cpus, n.CPUCount)
|
||||
}
|
||||
}
|
||||
f.CPUCores = catalog.SuggestCores(cpus)
|
||||
}
|
||||
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
|
||||
f.Warning = strings.TrimSuffix(f.Warning, ".") +
|
||||
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
|
||||
@@ -277,6 +289,11 @@ func buildModel(get func(string) string) (catalog.Model, error) {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
if v := strings.TrimSpace(get("cpu_cores")); v != "" {
|
||||
if p.CPUCores, err = positive(v, "the cores"); err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" {
|
||||
// What keeps a large model off the small nodes. A CPU profile has no
|
||||
// card to be short of, so without this every ready node looks like a
|
||||
|
||||
@@ -45,6 +45,13 @@ type DashboardData struct {
|
||||
|
||||
HasPeople bool
|
||||
|
||||
// NodePrefix is what every node's name starts with, left off in tables
|
||||
// so a row fits on a line; the full name is on hover.
|
||||
NodePrefix string
|
||||
// Columns shown only when some row has something in them: a CPU-only
|
||||
// cluster's nodes table was three columns of dashes wide.
|
||||
AnyGPUs, AnyClasses, AnyFabrics, AnyFallbacks bool
|
||||
|
||||
// Charts are pre-rendered rather than assembled in the template: building
|
||||
// SVG out of template fragments is how a chart ends up unescaped.
|
||||
RequestChart template.HTML
|
||||
@@ -177,6 +184,17 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
}
|
||||
}
|
||||
d.Unplaced = unplaced(src, d.Models)
|
||||
ids := make([]string, 0, len(d.Nodes))
|
||||
for _, n := range d.Nodes {
|
||||
ids = append(ids, n.ID)
|
||||
d.AnyGPUs = d.AnyGPUs || n.GPUTotal > 0
|
||||
d.AnyClasses = d.AnyClasses || len(n.Classes) > 0
|
||||
d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
|
||||
}
|
||||
d.NodePrefix = sharedPrefix(ids)
|
||||
for _, m := range d.Models {
|
||||
d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
|
||||
}
|
||||
d.Summary = src.Summary()
|
||||
if fs, ok := src.(FabricSource); ok {
|
||||
r := fs.Fabrics()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// cpuFleet is a CPU-only cluster whose nodes share a name prefix.
|
||||
type cpuFleet struct{ fakeSource }
|
||||
|
||||
func (cpuFleet) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{
|
||||
{ID: "llmgw-node01", State: "ready", MgmtAddr: "10.0.0.141", MemoryGB: 8},
|
||||
{ID: "llmgw-node02", State: "ready", MgmtAddr: "10.0.0.142", MemoryGB: 8},
|
||||
}
|
||||
}
|
||||
func (cpuFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
|
||||
func (cpuFleet) Models() []adminapi.ModelView {
|
||||
return []adminapi.ModelView{{Name: "qwen", Min: 1, Max: 1, Ready: 1,
|
||||
Description: "A sentence long enough that it once had a row of its own under the model."}}
|
||||
}
|
||||
func (cpuFleet) Replicas(string) []adminapi.ReplicaView {
|
||||
return []adminapi.ReplicaView{{ID: "qwen-1", Model: "qwen", State: "ready",
|
||||
Shards: []adminapi.ShardView{{Node: "llmgw-node02"}}}}
|
||||
}
|
||||
|
||||
func TestTheClusterPageKeepsEachRecordOnOneLine(t *testing.T) {
|
||||
h := Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil)
|
||||
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
|
||||
for _, gone := range []string{"<th>Classes</th>", "<th>Fabrics</th>", `<th class="num">GPUs</th>`, "<th>Fallbacks</th>"} {
|
||||
if strings.Contains(body, gone) {
|
||||
t.Errorf("%s is shown with nothing in it on any row", gone)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(body, `title="llmgw-node01"><code>node01</code>`) {
|
||||
t.Error("node names should drop what every node shares, keeping the whole on hover")
|
||||
}
|
||||
if !strings.Contains(body, `<code title="llmgw-node02">node02</code>`) {
|
||||
t.Error("the replicas table should name nodes the same way")
|
||||
}
|
||||
if strings.Contains(body, `class="url-row"`) || !strings.Contains(body, `<td class="clip" title="A sentence long enough`) {
|
||||
t.Error("a model's description should be a clipped cell, not a row of its own")
|
||||
}
|
||||
|
||||
// A cluster with GPUs still gets its GPU column.
|
||||
_, gpu := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
if !strings.Contains(gpu, `<th class="num">GPUs</th>`) || !strings.Contains(gpu, "<th>Fabrics</th>") {
|
||||
t.Error("columns with something in them were hidden")
|
||||
}
|
||||
}
|
||||
@@ -837,3 +837,21 @@ a.open-held[aria-expanded=false]::before { content: "▸ "; color: var(--muted);
|
||||
.use .runs-now.short { color: var(--warn); }
|
||||
.use .runs-now.bad { color: var(--bad); }
|
||||
.use .runs-now .node { white-space: nowrap; }
|
||||
|
||||
/* One line per record, in every table. A header never wraps; a cell of free
|
||||
* text is cut to its column with the whole of it on hover, instead of a row of
|
||||
* its own or a line break that makes one record look like two. */
|
||||
th { white-space: nowrap; }
|
||||
table.rows td { white-space: nowrap; }
|
||||
td.clip { max-width: 22rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
table.rows td.clip { color: var(--muted); font-size: .85rem; }
|
||||
/* A row across the whole table is a note or an open form, not a record: it
|
||||
* wraps as prose does. */
|
||||
table.rows td[colspan] { white-space: normal; }
|
||||
table.rows td.where { max-width: none; }
|
||||
table.rows .more { cursor: help; margin-left: .2rem; }
|
||||
table.explore td:first-child { max-width: 34rem; overflow: hidden; text-overflow: ellipsis; }
|
||||
table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted); }
|
||||
.clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis;
|
||||
vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; }
|
||||
table.rows td.warn { color: var(--warn); cursor: help; }
|
||||
@@ -84,7 +84,7 @@ Tell whoever you asked for access that you have signed up.</p>
|
||||
{{end}}
|
||||
{{if .Fields}}
|
||||
<h3>Settings to fill in</h3>
|
||||
<table class="compact narrow-table connect-fields">{{range .Fields}}
|
||||
<table class="rows compact narrow-table connect-fields">{{range .Fields}}
|
||||
<tr><th>{{.Label}}</th><td><code>{{.Value}}</code></td><td><button type="button" class="link copy" data-value="{{.Value}}">copy</button></td></tr>{{end}}
|
||||
</table>
|
||||
{{end}}
|
||||
@@ -149,7 +149,7 @@ document.addEventListener("click", (e) => {
|
||||
|
||||
<h2>Keys</h2>
|
||||
{{if .Keys}}
|
||||
<table><tr><th>Name</th><th>Key</th><th>Spends against</th><th>Created</th><th>Last used</th><th>State</th><th></th></tr>
|
||||
<table class="rows"><tr><th>Name</th><th>Key</th><th>Spends against</th><th>Created</th><th>Last used</th><th>State</th><th></th></tr>
|
||||
{{range .Keys}}<tr>
|
||||
<td>{{.Name}}</td>
|
||||
<td><code>{{.Prefix}}</code></td>
|
||||
@@ -192,7 +192,7 @@ old one to stop moving, then revoke it.</p>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if .Rows}}
|
||||
<table class="compact narrow-table"><tr><th>Limit</th><th class="num">Value</th></tr>
|
||||
<table class="rows compact narrow-table"><tr><th>Limit</th><th class="num">Value</th></tr>
|
||||
{{range .Rows}}<tr><td>{{.Name}}</td><td class="num">{{.Value}}</td></tr>{{end}}</table>
|
||||
{{else}}<p class="empty">No limits here.</p>{{end}}
|
||||
{{else}}<p class="empty">Nowhere to spend yet.</p>{{end}}
|
||||
|
||||
@@ -34,7 +34,7 @@ reachable from this machine.</p>
|
||||
|
||||
<div id="held">
|
||||
{{if .Rows}}
|
||||
<table class="compact held">
|
||||
<table class="rows compact held">
|
||||
<tr><th>Model</th><th>Files</th><th class="num">Size</th><th>Runs</th><th></th></tr>
|
||||
{{range .Groups}}{{$grouped := gt .Models 1}}
|
||||
{{if $grouped}}
|
||||
@@ -165,7 +165,9 @@ reachable from this machine.</p>
|
||||
<div class="field-row">
|
||||
<label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
|
||||
<label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
|
||||
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}"></label>
|
||||
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}" required></label>
|
||||
<label>CPU cores <span class="opt">reserved; the engine's threads</span>
|
||||
<input type="text" name="cpu_cores" value="{{.CPUCores}}"></label>
|
||||
<label>Only nodes labelled <span class="opt">key / value, optional</span>
|
||||
<input type="text" name="selector_key" placeholder="role" size="8">
|
||||
<input type="text" name="selector_value" placeholder="cpu-large" size="10"></label>
|
||||
@@ -239,13 +241,13 @@ document.addEventListener('click', function (e) {
|
||||
|
||||
{{if not .Repo}}{{with .Explore}}
|
||||
{{if .Rows}}
|
||||
<table class="explore">
|
||||
<table class="rows explore">
|
||||
<tr><th>Repository</th><th class="num">Size</th><th>Architecture</th><th class="num">Context</th>
|
||||
<th class="num">Downloads</th><th class="num">Likes</th><th class="num">Updated</th></tr>
|
||||
{{range .Rows}}<tr class="explore-row">
|
||||
<td><a href="{{.Href}}" class="open-repo" data-repo="{{.ID}}" aria-expanded="false"><code>{{.ID}}</code></a>
|
||||
{{if .New}}<span class="tag new">new</span>{{end}}{{if .Gated.Is}}<span class="tag">gated</span>{{end}}
|
||||
{{if or .BaseModel .License}}<div class="from">{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}</div>{{end}}</td>
|
||||
{{if or .BaseModel .License}}<span class="from" title="{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}">{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}</span>{{end}}</td>
|
||||
<td class="num">{{.ParamsLabel}}</td>
|
||||
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{.ContextLabel}}</td>
|
||||
@@ -399,7 +401,7 @@ version and the old weights stay where they are.</p>
|
||||
</details>
|
||||
{{end}}
|
||||
{{if .RepoSets}}
|
||||
<table class="repo-files">
|
||||
<table class="rows repo-files">
|
||||
<tr><th>What</th><th class="num">Size</th><th>Published checksum</th><th></th></tr>
|
||||
{{range .RepoSets}}<tr>
|
||||
<td><code>{{.Name}}</code>
|
||||
@@ -409,7 +411,7 @@ version and the old weights stay where they are.</p>
|
||||
<td>
|
||||
{{if not .Verdict.Servable}}
|
||||
<span class="state state-failed" title="{{.Verdict.Reason}}">cannot be served</span>
|
||||
<div class="note">{{.Verdict.Reason}}</div>
|
||||
<span class="clip-inline" title="{{.Verdict.Reason}}">{{.Verdict.Reason}}</span>
|
||||
{{else}}
|
||||
{{if .Verdict.Architecture}}<span class="opt" title="{{.Verdict.Reason}}">{{.Verdict.Architecture}}{{if .Verdict.Checked}} ✓{{end}}</span>{{end}}
|
||||
<form method="post" action="/admin/mirror/fetch" class="inline">
|
||||
@@ -462,7 +464,7 @@ computes once it has it.</p>
|
||||
{{with .View.Reason}}<p class="note error-row">{{.}}</p>{{end}}
|
||||
|
||||
{{if .Replicas}}
|
||||
<table class="compact replicas">
|
||||
<table class="rows compact replicas">
|
||||
<tr><th>Replica</th><th>Node</th><th>State</th><th class="num">Age</th><th class="num">CPU</th><th class="num">Memory</th></tr>
|
||||
{{range .Replicas}}{{$rep := .}}{{range $i, $s := .Nodes}}<tr>
|
||||
<td>{{if eq $i 0}}<code>{{$rep.ID}}</code>{{end}}</td>
|
||||
|
||||
@@ -34,7 +34,7 @@ budgets bind.</p>
|
||||
{{/* The tree is the interface. A quota belongs to a branch, so the form to set
|
||||
one opens on that branch rather than asking somebody to name it again in a
|
||||
box at the bottom. */}}
|
||||
<table class="quota-tree">
|
||||
<table class="rows quota-tree">
|
||||
<tr><th>Where</th><th class="num">Requests/min</th><th class="num">Tokens/min</th>
|
||||
<th class="num">Budget</th><th>Spent</th><th></th></tr>
|
||||
{{range .Rows}}
|
||||
@@ -119,7 +119,7 @@ up — hover it to see where. A value set here is in normal type.</p>
|
||||
<p class="note">A person's own limit covers everything they do, across all their
|
||||
keys, projects and the chat page. {{.Personal}}</p>
|
||||
{{if .People}}
|
||||
<table class="quota-tree people-limits">
|
||||
<table class="rows quota-tree people-limits">
|
||||
<tr><th>Person</th><th class="num">Requests/min</th><th class="num">Tokens/min</th>
|
||||
<th class="num">Budget</th><th>Spent</th><th></th></tr>
|
||||
{{range .People}}
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
</div>
|
||||
|
||||
{{if .Rows}}
|
||||
<table>
|
||||
<table class="rows">
|
||||
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
|
||||
{{range .Rows}}
|
||||
<tr>
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
{{if .Pending}}
|
||||
<h2>Waiting for approval</h2>
|
||||
<table><tr><th>Email</th><th>Name</th><th>Signed up</th><th>Approve, and put them in</th><th></th></tr>
|
||||
<table class="rows"><tr><th>Email</th><th>Name</th><th>Signed up</th><th>Approve, and put them in</th><th></th></tr>
|
||||
{{range .Pending}}<tr>
|
||||
<td>{{.Email}}</td><td>{{.Name}}</td><td>{{ago .CreatedAt}}</td>
|
||||
<td>
|
||||
@@ -149,14 +149,14 @@ project in the team — an account can only be approved into a project.</p>
|
||||
</div>
|
||||
|
||||
{{with index $.Members (printf "project:%s" $p.ID)}}
|
||||
<table class="compact"><tr><th>Member</th><th>State</th><th>Last sign-in</th><th></th></tr>
|
||||
<table class="rows compact"><tr><th>Member</th><th>State</th><th>Last sign-in</th><th></th></tr>
|
||||
{{range .}}{{template "member-row" (dict "U" . "D" $ "Scope" (printf "project:%s" $p.ID) "Open" $p.ID)}}{{end}}
|
||||
</table>
|
||||
{{else}}<p class="empty">No members of its own. Members of its team or organization can use it too.</p>{{end}}
|
||||
{{template "add-member" (dict "Kind" "project" "ID" $p.ID "D" $ "Open" $p.ID)}}
|
||||
|
||||
{{with index $.Keys $p.ID}}
|
||||
<table class="compact"><tr><th>Key</th><th>Owner</th><th>State</th><th>Last used</th><th></th></tr>
|
||||
<table class="rows compact"><tr><th>Key</th><th>Owner</th><th>State</th><th>Last used</th><th></th></tr>
|
||||
{{range .}}<tr>
|
||||
<td>{{.Name}} <code>{{.Prefix}}</code></td>
|
||||
<td>{{or .OwnerEmail "issued by an operator"}}</td>
|
||||
@@ -210,7 +210,7 @@ project in the team — an account can only be approved into a project.</p>
|
||||
|
||||
<h2 id="accounts">All accounts</h2>
|
||||
{{if .Accounts}}
|
||||
<table id="account-list"><tr><th>Person</th><th>State</th><th>Member of</th><th>Last sign-in</th><th>Add to</th><th></th></tr>
|
||||
<table class="rows" id="account-list"><tr><th>Person</th><th>State</th><th>Member of</th><th>Last sign-in</th><th>Add to</th><th></th></tr>
|
||||
{{range .Accounts}}{{template "account-row" (dict "U" . "D" $)}}{{end}}
|
||||
</table>
|
||||
<p class="note">Someone who is a member of nothing uses the gateway on their own:
|
||||
@@ -298,7 +298,7 @@ access than it had, so they need a shell on the control plane —
|
||||
<td>{{$u.Email}}{{if $u.IsAdmin}} <span class="tag">admin</span>{{end}}</td>
|
||||
<td class="state state-{{$u.State}}">{{$u.State}}</td>
|
||||
<td class="where">
|
||||
{{range $u.Memberships}}<span class="chip-m">{{.Path}}{{if eq .Role "manager"}} <b class="chip-role">manager</b>{{end}}
|
||||
{{range $i, $m := $u.Memberships}}{{if lt $i 2}}{{with $m}}<span class="chip-m">{{.Path}}{{if eq .Role "manager"}} <b class="chip-role">manager</b>{{end}}
|
||||
{{if eq (printf "%s" .Scope) "org"}}
|
||||
<form method="post" action="/admin/users/{{$u.ID}}/role" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$d.CSRF}}">
|
||||
@@ -313,8 +313,12 @@ access than it had, so they need a shell on the control plane —
|
||||
<input type="hidden" name="csrf" value="{{$d.CSRF}}">
|
||||
<input type="hidden" name="scope" value="{{.Scope}}:{{.ScopeID}}">
|
||||
<button type="submit" class="chip-x" aria-label="Take out of {{.Path}}" title="Take out of {{.Path}}">×</button>
|
||||
</form></span>
|
||||
</form></span>{{end}}{{end}}
|
||||
{{else}}<span class="opt">on their own</span>{{end}}
|
||||
{{/* Two, then a count: a person in six teams was six chips wrapping the
|
||||
row into a paragraph. The rest are named on hover and in their own
|
||||
scopes' member lists, where taking them out is done. */}}
|
||||
{{if gt (len $u.Memberships) 2}}<span class="opt more" title="{{range $i, $m := $u.Memberships}}{{if ge $i 2}}{{$m.Path}} {{end}}{{end}}">+{{sub (len $u.Memberships) 2}}</span>{{end}}
|
||||
</td>
|
||||
<td>{{ago $u.LastLoginAt}}</td>
|
||||
<td>
|
||||
@@ -365,7 +369,7 @@ access than it had, so they need a shell on the control plane —
|
||||
<details class="edit"><summary>{{len (index $d.Members $k)}} member{{if ne (len (index $d.Members $k)) 1}}s{{end}}</summary>
|
||||
<div class="scope-members">
|
||||
{{with index $d.Members $k}}
|
||||
<table class="compact">
|
||||
<table class="rows compact">
|
||||
{{range .}}{{template "member-row" (dict "U" . "D" $d "Scope" $k "Open" "")}}{{end}}
|
||||
</table>
|
||||
{{else}}<p class="note">Nobody is a member of {{.Name}} itself; a member here could use every project in it.</p>{{end}}
|
||||
|
||||
@@ -51,23 +51,23 @@
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{if .Nodes}}
|
||||
<table class="nodes"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th><th class="num">GPUs</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||||
<table class="nodes rows"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">Disk</th><th>Engines</th>{{if .AnyClasses}}<th>Classes</th>{{end}}{{if .AnyFabrics}}<th>Fabrics</th>{{end}}<th class="num">Models</th><th class="num">Replicas</th><th>Seen</th></tr>
|
||||
{{range .Nodes}}<tr>
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}"><code>{{.ID}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}" title="{{.ID}}"><code>{{shortNode .ID $.NodePrefix}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
{{/* A dash, not an empty bar, when the agent has not measured it lately:
|
||||
unknown is not idle. */}}
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} <span class="note">of {{.CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">of {{bytes .MemTotal}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||||
<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>
|
||||
{{if $.AnyGPUs}}<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>{{end}}
|
||||
{{/* Free out of total, because the first is meaningless without the
|
||||
second, and an em dash where nothing was reported: an agent that
|
||||
was never asked about its disk must not read as a full one. */}}
|
||||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">none yet</span>{{end}}</td>
|
||||
<td>{{join .Classes}}</td>
|
||||
<td>{{join .Fabrics}}</td>
|
||||
{{if $.AnyClasses}}<td>{{join .Classes}}</td>{{end}}
|
||||
{{if $.AnyFabrics}}<td>{{join .Fabrics}}</td>{{end}}
|
||||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{.Replicas}}</td>
|
||||
<td>{{or .LastSeenAgo "—"}}</td>
|
||||
@@ -80,18 +80,20 @@
|
||||
<h2>Models</h2>
|
||||
|
||||
{{if .Models}}
|
||||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||||
<table class="rows"><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Reserves</th><th>Profiles</th>{{if .AnyFallbacks}}<th>Fallbacks</th>{{end}}<th>What it is for</th></tr>
|
||||
{{range .Models}}<tr>
|
||||
<td><code>{{.Name}}</code></td>
|
||||
<td>{{join .Aliases}}</td>
|
||||
<td class="num">{{.Ready}}</td>
|
||||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||||
<td{{with .Warnings}} class="warn" title="{{join .}}"{{end}}>{{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}}
|
||||
{{- if .MeasuredMemoryGB}} <span class="note">{{printf "%.1f" .MeasuredMemoryGB}}G used</span>{{end}}{{if .Warnings}} ⚠{{end}}</td>
|
||||
<td>{{join .Profiles}}</td>
|
||||
<td>{{join .Fallbacks}}</td>
|
||||
{{if $.AnyFallbacks}}<td>{{join .Fallbacks}}</td>{{end}}
|
||||
{{/* Cut to the column, the whole sentence on hover: a row of its own for
|
||||
each description made the table twice as long as the catalog. */}}
|
||||
<td class="clip"{{with .Description}} title="{{.}}"{{end}}>{{or .Description "—"}}</td>
|
||||
</tr>
|
||||
{{/* Under the row rather than in a column: a sentence in a table cell either
|
||||
wraps every other column into a stripe or gets cut off. */}}
|
||||
{{if .Description}}<tr><td colspan="6" class="url-row">{{.Description}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||||
|
||||
@@ -106,7 +108,7 @@
|
||||
|
||||
<h2>Replicas</h2>
|
||||
{{if .Replicas}}
|
||||
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||||
<table class="rows"><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||||
{{range .Replicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
@@ -114,7 +116,7 @@
|
||||
<td>{{or .Pool "—"}}</td>
|
||||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||||
<td class="num">{{duration .AgeSec}}</td>
|
||||
<td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
|
||||
<td>{{range .Shards}}<code title="{{.Node}}">{{shortNode .Node $.NodePrefix}}</code> {{end}}</td>
|
||||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
<h2>Fabric</h2>
|
||||
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}} · <a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">the map →</a>{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
|
||||
{{if .Node.IBPorts}}
|
||||
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
<table class="rows"><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
{{range .Node.IBPorts}}<tr>
|
||||
<td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag">RoCE</span>{{end}}</td>
|
||||
<td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
|
||||
@@ -60,13 +60,13 @@ the adapter came up: a number that grows is the one to look at.</p>
|
||||
{{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}<span class="state-ready">active</span>, {{.Node.IMEXChannels}} channel{{if ne .Node.IMEXChannels 1}}s{{end}}{{else}}<span class="bad">{{or .Node.IMEX "not installed"}}</span>{{end}}{{end}}
|
||||
</p>
|
||||
<div class="gpu-wiring">
|
||||
<table class="nvl-matrix" aria-label="NVLink between GPUs">
|
||||
<table class="rows nvl-matrix" aria-label="NVLink between GPUs">
|
||||
<tr><th></th>{{range .Node.GPUs}}<th>{{.Index}}</th>{{end}}</tr>
|
||||
{{range $a := .Node.GPUs}}<tr><th>GPU {{$a.Index}}</th>
|
||||
{{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}<td class="self"></td>{{else if hasInt $a.NVLinkPeers $b.Index}}<td class="nv" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over NVLink">NV</td>{{else}}<td class="pcie" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over PCIe">·</td>{{end}}{{end}}
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
<table class="compact gpu-list">
|
||||
<table class="rows compact gpu-list">
|
||||
<tr><th>GPU</th><th>Name</th><th class="num">NUMA</th><th>Nearest adapter</th><th>Multi-node NVLink</th></tr>
|
||||
{{range .Node.GPUs}}<tr>
|
||||
<td>{{.Index}}</td><td>{{.Name}}</td><td class="num">{{.NUMA}}</td>
|
||||
@@ -82,7 +82,7 @@ traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS
|
||||
|
||||
<h2>What runs here</h2>
|
||||
{{if .Engines}}
|
||||
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
||||
<table class="rows"><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
||||
{{range .Engines}}<tr>
|
||||
<td>{{.Model}}</td>
|
||||
<td><code>{{.Replica}}</code></td>
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
{{end}}
|
||||
|
||||
{{if .Rows}}
|
||||
<table class="compact">
|
||||
<table class="rows compact">
|
||||
<tr><th>{{if eq .By "user"}}Person{{else if eq .By "project"}}Project{{else}}Team{{end}}</th>
|
||||
<th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
|
||||
{{range .Rows}}<tr>
|
||||
@@ -56,7 +56,7 @@ people do elsewhere — in another organization, or on their own — is not show
|
||||
|
||||
<h2>Teams, projects and limits</h2>
|
||||
{{if .Quotas}}
|
||||
<table class="quota-tree">
|
||||
<table class="rows quota-tree">
|
||||
<tr><th>Where</th><th class="num">Requests/min</th><th class="num">Tokens/min</th>
|
||||
<th class="num">Budget</th><th>Spent · 30 days</th></tr>
|
||||
{{range .Quotas}}
|
||||
@@ -79,7 +79,7 @@ people do elsewhere — in another organization, or on their own — is not show
|
||||
|
||||
<h2>Members</h2>
|
||||
{{if .Members}}
|
||||
<table class="compact">
|
||||
<table class="rows compact">
|
||||
<tr><th>Person</th><th>Member of</th><th>State</th></tr>
|
||||
{{range .Members}}<tr>
|
||||
<td>{{.Email}}</td>
|
||||
|
||||
+3
-1
@@ -109,7 +109,9 @@ var funcs = template.FuncMap{
|
||||
return p
|
||||
},
|
||||
"ago": ago,
|
||||
"age": age,
|
||||
// shortNode is a node's name without what every node's name starts with.
|
||||
"shortNode": strings.TrimPrefix,
|
||||
"age": age,
|
||||
// pctbar is a percentage as the minibar template takes it: whole, and
|
||||
// held to 0..100 so a measurement a little over does not draw past its box.
|
||||
"pctbar": func(v float64) struct{ P int } {
|
||||
|
||||
Reference in new issue
Block a user