A replica says what its engine computes on, and llama.cpp's word on the CPU refuses a GPU placement
llama.cpp's start-up output is read as a shard becomes ready: the devices it used, its backend, and the layers it offloaded. That goes up with READY in a new ShardStatus.compute, is kept on the shard, and is shown by the admin API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose engine says it found no usable GPU or offloaded no layer is stopped and failed, even where nvidia-smi cannot be asked. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f285aa0548
commit
0bd0d881c5
15 files changed
+517
-126
No files matched your search
+10
-4
@@ -450,7 +450,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tON\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
for _, r := range reps {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
@@ -468,8 +468,14 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if r.Restart != "" {
|
||||
state += " (restart " + r.Restart + ")"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, state, r.Pool, join(nodes),
|
||||
// What the engine computes on, by its own account: the first
|
||||
// shard's, which serves.
|
||||
on := "-"
|
||||
if len(r.Shards) > 0 {
|
||||
on = r.Shards[0].Compute.Short()
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, state, r.Pool, on, join(nodes),
|
||||
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||
idleColumn(r.IdleSec), r.Endpoint)
|
||||
// A line rather than a column, because most replicas are not
|
||||
@@ -479,7 +485,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if s.Fetch == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\n",
|
||||
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\t\n",
|
||||
s.Node, fetchColumn(s.Fetch))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1230,13 +1230,17 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||||
}
|
||||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||||
compute := "-"
|
||||
if len(r.Shards) > 0 {
|
||||
compute = r.Shards[0].Compute.Short()
|
||||
}
|
||||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, compute, strings.Join(nodes, " "),
|
||||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||||
}
|
||||
if m.inPanel {
|
||||
keys = rowKeys
|
||||
}
|
||||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "COMPUTE", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||||
case cUsage:
|
||||
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
|
||||
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
|
||||
|
||||
Reference in new issue
Block a user