Every setting of a served model from the mirror page, and its replicas per profile
The row of a served model leads to its card, which now changes its context and longest answer, its cards, its replicas on each profile with a profile on CPUs to add and its speed beside, stops, starts and restarts it, and removes it with its files or without. Stopped on the row stops the model rather than setting its scaling to nothing. The console gains Replicas per profile, and ctl models shows the counts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e23732fbbb
commit
1a3f385be4
15 files changed
+721
-20
No files matched your search
+14
-1
@@ -418,7 +418,7 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m),
|
||||
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
|
||||
join(m.Aliases), join(profilesWithCounts(m)), join(m.Fallbacks))
|
||||
for _, x := range m.Warnings {
|
||||
warnings = append(warnings, m.Name+": "+x)
|
||||
}
|
||||
@@ -696,3 +696,16 @@ func reserves(m adminapi.ModelView) string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// profilesWithCounts are a model's profiles, each with its replicas when
|
||||
// they are counted per profile: gpu=7, cpu=4.
|
||||
func profilesWithCounts(m adminapi.ModelView) []string {
|
||||
if len(m.ProfileReplicas) == 0 {
|
||||
return m.Profiles
|
||||
}
|
||||
out := make([]string, len(m.Profiles))
|
||||
for i, p := range m.Profiles {
|
||||
out[i] = fmt.Sprintf("%s=%d", p, m.ProfileReplicas[p])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
@@ -406,6 +407,48 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"Replicas per profile…", func(m *consoleModel) tea.Cmd {
|
||||
var now []string
|
||||
for _, p := range md.Profiles {
|
||||
now = append(now, fmt.Sprintf("%s=%d", p, md.ProfileReplicas[p]))
|
||||
}
|
||||
m.openForm(&formDialog{title: "Replicas per profile of " + md.Name,
|
||||
note: "How many replicas run on each profile, all answering under the model's name: seven on cards and " +
|
||||
"four on CPUs, the router filling the cards first. A profile not named keeps its count; cpu=N with " +
|
||||
"the memory below adds a profile on CPUs to a GGUF model that has none. Those short start first, " +
|
||||
"then those over stop.",
|
||||
fields: []formField{
|
||||
textF("replicas", strings.Join(now, " "), "profile=count, separated by spaces"),
|
||||
textF("CPU memory GB", "", "only to add a profile on CPUs: what a replica reserves"),
|
||||
textF("CPU cores", "", "of a replica on CPUs; empty is half the node's"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
req := adminapi.ProfileReplicasRequest{Profiles: map[string]int{}}
|
||||
for _, f := range strings.Fields(v[0]) {
|
||||
name, c, ok := strings.Cut(f, "=")
|
||||
n, err := strconv.Atoi(c)
|
||||
if !ok || err != nil || n < 0 || name == "" {
|
||||
return nil, fmt.Errorf("%q: a profile and a count, as gpu=7", f)
|
||||
}
|
||||
req.Profiles[name] = n
|
||||
}
|
||||
if v[1] != "" {
|
||||
mem, err := strconv.Atoi(v[1])
|
||||
if err != nil || mem < 1 {
|
||||
return nil, fmt.Errorf("the CPU memory is a whole number of GB")
|
||||
}
|
||||
cores := 0
|
||||
if v[2] != "" {
|
||||
if cores, err = strconv.Atoi(v[2]); err != nil || cores < 1 {
|
||||
return nil, fmt.Errorf("the CPU cores are a whole number from 1")
|
||||
}
|
||||
}
|
||||
req.CPU = &modelsvc.CPUProfile{MemoryGB: mem, CPUCores: cores}
|
||||
}
|
||||
return m.call("Replicas of "+md.Name, http.MethodPost, path+"/replicas", req, nil), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
{"Resources…", func(m *consoleModel) tea.Cmd {
|
||||
profiles := md.Profiles
|
||||
if len(profiles) == 0 {
|
||||
|
||||
Reference in new issue
Block a user