A model can be stopped and kept, or removed with its files from the mirror and the nodes

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 20:45:48 +01:00
1 parent 5726ceb84e
commit 59187cdae5
19 files changed
+760 -14

No files matched your search

+2
View File
@@ -321,6 +321,8 @@ func locationColumn(n adminapi.NodeView) string {
// actually served, never what is merely catalogued.
func modelStatus(m adminapi.ModelView) string {
switch {
case m.Stopped:
return "stopped (llmgw ctl model start)"
case m.Parked:
return "parked"
case m.External != "":
+90 -7
View File
@@ -34,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
c.AddCommand(newModelStopCmd(client), newModelStartCmd(client), newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -163,21 +163,104 @@ func readModelFile(path string) (catalog.Model, error) {
}
func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
var purge bool
c := &cobra.Command{
Use: "remove <name>",
Short: "Remove a model that was added at runtime",
Short: "Remove a model that was added at runtime, and with --purge its files",
Long: "Only models added at runtime. One declared in server.yaml would come\n" +
"back on the next restart, so removing it here would leave the cluster\n" +
"disagreeing with its own configuration until then.\n\n" +
"Its replicas are stopped on the next pass, after the endpoint has been\n" +
"withdrawn, so traffic stops before the engine does.",
"withdrawn, so traffic stops before the engine does.\n\n" +
"Its files stay on the mirror and the nodes, for adding it again. With\n" +
"--purge they go too: every version of it on the mirror (a mirror kept in\n" +
"step follows), and the nodes' copies once its replicas have stopped.\n" +
"What another model uses stays. To take a model out of service and keep\n" +
"it, use llmgw ctl model stop.",
Example: " llmgw ctl model remove qwen-7b\n" +
" llmgw ctl model remove qwen-7b --purge # its files on the mirror and the nodes too",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var res map[string]string
if err := client().delete(cmd.Context(), "/api/v1/models/"+args[0], &res); err != nil {
out := cmd.OutOrStdout()
if !purge {
var res map[string]string
if err := client().delete(cmd.Context(), "/api/v1/models/"+args[0], &res); err != nil {
return err
}
fmt.Fprintf(out, "%s removed. Its files stay on the mirror and the nodes; --purge deletes them.\n", args[0])
return nil
}
var v adminapi.PurgeView
if err := client().patient(3*time.Minute).delete(cmd.Context(), "/api/v1/models/"+args[0]+"?purge=true", &v); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s removed.\n", args[0])
fmt.Fprintf(out, "%s removed.\n", v.Removed)
for _, m := range v.Mirror {
fmt.Fprintf(out, " mirror %s deleted\n", m)
}
for _, m := range v.Kept {
fmt.Fprintf(out, " mirror %s kept: another model uses it\n", m)
}
var freed int64
for _, n := range v.Nodes {
if n.Error != "" {
fmt.Fprintf(out, " node %s: %s\n", n.Node, n.Error)
continue
}
freed += n.Bytes
fmt.Fprintf(out, " node %s: %d file(s), %s\n", n.Node, n.Files, humanBytes(n.Bytes))
}
if len(v.Mirror) == 0 && len(v.Nodes) == 0 && v.Note == "" {
fmt.Fprintln(out, " it had no files of its own on the mirror or the nodes")
}
if freed > 0 {
fmt.Fprintf(out, " %s freed on the nodes\n", humanBytes(freed))
}
if v.Note != "" {
fmt.Fprintln(out, " "+v.Note)
}
return nil
},
}
c.Flags().BoolVar(&purge, "purge", false, "delete its files from the mirror and the nodes as well")
return c
}
func newModelStopCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "stop <model>",
Short: "Take a model out of service and keep it: none of its replicas run until it is started",
Long: "Its replicas stop; it stays in the catalog with its files, and a request\n" +
"for it is told an administrator stopped it. What it was scaled to is kept:\n" +
"llmgw ctl model start runs it again as it was.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var v adminapi.ScaleResponse
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/stop", nil, &v); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s stopped: its replicas stop on the next pass. It is kept; "+
"llmgw ctl model start %s runs it again.\n", v.Model, v.Model)
return nil
},
}
}
func newModelStartCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "start <model>",
Short: "Run a stopped model again, scaled as it was when it was stopped",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var v adminapi.ScaleResponse
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/start", nil, &v); err != nil {
return err
}
how := fmt.Sprintf("%d replica(s)", v.Min)
if v.OnDemand {
how = "on demand"
}
fmt.Fprintf(cmd.OutOrStdout(), "%s started: %s, max %d. Watch it with llmgw ctl models.\n", v.Model, how, v.Max)
return nil
},
}
+4
View File
@@ -583,6 +583,10 @@ func newServerCmd() *cobra.Command {
agentsvc.SetSourceRewrite(src.Rewrite)
}
rt.SetMirror(mc)
if adminAPI != nil {
// A model removed with its files is removed from it too.
adminAPI.SetPurgeMirror(mc)
}
if fwPub != nil {
fwPub.setMirror(mc)
}
+68 -2
View File
@@ -97,10 +97,21 @@ func (m *consoleModel) openForm(f *formDialog) {
// call runs one admin API request in the background and reports it.
func (m *consoleModel) call(title, method, path string, body any, say func(raw json.RawMessage) string) tea.Cmd {
return m.callWaiting(title, method, path, body, say, 60*time.Second)
}
// callWaiting is call for a change the server may spend longer on than the
// console's usual wait.
func (m *consoleModel) callWaiting(title, method, path string, body any, say func(raw json.RawMessage) string,
wait time.Duration,
) tea.Cmd {
m.busy = title
c := m.client
if wait > 60*time.Second {
c = c.patient(wait)
}
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), wait)
defer cancel()
var raw json.RawMessage
// A change may answer with no body at all.
@@ -479,13 +490,40 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
}})
}
if md.Stopped {
items = append(items, menuItem{"Start", func(m *consoleModel) tea.Cmd {
return m.call("Start "+md.Name, http.MethodPost, path+"/start", nil, func(json.RawMessage) string {
return md.Name + " runs again, scaled as it was when it was stopped."
})
}})
} else {
items = append(items, menuItem{"Stop…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Stop " + md.Name,
text: "Its replicas stop; it stays in the catalog with its files, and Start runs it again as it was.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Stop "+md.Name, http.MethodPost, path+"/stop", nil, func(json.RawMessage) string {
return md.Name + " is stopped and kept."
})
}}
return nil
}})
}
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Remove " + md.Name,
text: "Its replicas stop and it leaves the catalog. A model from server.yaml cannot be removed here.",
text: "Its replicas stop and it leaves the catalog; its files stay on the mirror and the nodes. " +
"A model from server.yaml cannot be removed here.",
yes: func(m *consoleModel) tea.Cmd {
return m.call("Remove "+md.Name, http.MethodDelete, path, nil, nil)
}}
return nil
}}, menuItem{"Remove with its files…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Remove " + md.Name + " and its files",
text: "It leaves the catalog, every version of it is deleted from the mirror, and the nodes' copies once " +
"its replicas have stopped. What another model uses stays. This cannot be undone: adding it again fetches it again.",
yes: func(m *consoleModel) tea.Cmd {
return m.callWaiting("Remove "+md.Name, http.MethodDelete, path+"?purge=true", nil, purgeSays, 3*time.Minute)
}}
return nil
}})
return &menuDialog{title: md.Name, items: items}
}
@@ -990,3 +1028,31 @@ func gpusSays(raw json.RawMessage) string {
}
return s
}
// purgeSays is a model removed with its files, in a few lines.
func purgeSays(raw json.RawMessage) string {
var v adminapi.PurgeView
if json.Unmarshal(raw, &v) != nil {
return "Done."
}
s := v.Removed + " removed."
if len(v.Mirror) > 0 {
s += fmt.Sprintf("\nMirror: %d version(s) deleted.", len(v.Mirror))
}
if len(v.Kept) > 0 {
s += "\nKept on the mirror, another model uses them: " + strings.Join(v.Kept, ", ")
}
var files int
var freed int64
for _, n := range v.Nodes {
files += n.Files
freed += n.Bytes
}
if files > 0 {
s += fmt.Sprintf("\nNodes: %d file(s), %s freed.", files, humanBytes(freed))
}
if v.Note != "" {
s += "\n" + v.Note
}
return s
}