A model can be stopped and kept, or removed with its files from the mirror and the nodes

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 20:45:48 +01:00
1 parent 5726ceb84e
commit 59187cdae5
19 files changed
+760 -14

No files matched your search

+90 -7
View File
@@ -34,7 +34,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
c.AddCommand(newModelStopCmd(client), newModelStartCmd(client), newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -163,21 +163,104 @@ func readModelFile(path string) (catalog.Model, error) {
}
func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
var purge bool
c := &cobra.Command{
Use: "remove <name>",
Short: "Remove a model that was added at runtime",
Short: "Remove a model that was added at runtime, and with --purge its files",
Long: "Only models added at runtime. One declared in server.yaml would come\n" +
"back on the next restart, so removing it here would leave the cluster\n" +
"disagreeing with its own configuration until then.\n\n" +
"Its replicas are stopped on the next pass, after the endpoint has been\n" +
"withdrawn, so traffic stops before the engine does.",
"withdrawn, so traffic stops before the engine does.\n\n" +
"Its files stay on the mirror and the nodes, for adding it again. With\n" +
"--purge they go too: every version of it on the mirror (a mirror kept in\n" +
"step follows), and the nodes' copies once its replicas have stopped.\n" +
"What another model uses stays. To take a model out of service and keep\n" +
"it, use llmgw ctl model stop.",
Example: " llmgw ctl model remove qwen-7b\n" +
" llmgw ctl model remove qwen-7b --purge # its files on the mirror and the nodes too",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var res map[string]string
if err := client().delete(cmd.Context(), "/api/v1/models/"+args[0], &res); err != nil {
out := cmd.OutOrStdout()
if !purge {
var res map[string]string
if err := client().delete(cmd.Context(), "/api/v1/models/"+args[0], &res); err != nil {
return err
}
fmt.Fprintf(out, "%s removed. Its files stay on the mirror and the nodes; --purge deletes them.\n", args[0])
return nil
}
var v adminapi.PurgeView
if err := client().patient(3*time.Minute).delete(cmd.Context(), "/api/v1/models/"+args[0]+"?purge=true", &v); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s removed.\n", args[0])
fmt.Fprintf(out, "%s removed.\n", v.Removed)
for _, m := range v.Mirror {
fmt.Fprintf(out, " mirror %s deleted\n", m)
}
for _, m := range v.Kept {
fmt.Fprintf(out, " mirror %s kept: another model uses it\n", m)
}
var freed int64
for _, n := range v.Nodes {
if n.Error != "" {
fmt.Fprintf(out, " node %s: %s\n", n.Node, n.Error)
continue
}
freed += n.Bytes
fmt.Fprintf(out, " node %s: %d file(s), %s\n", n.Node, n.Files, humanBytes(n.Bytes))
}
if len(v.Mirror) == 0 && len(v.Nodes) == 0 && v.Note == "" {
fmt.Fprintln(out, " it had no files of its own on the mirror or the nodes")
}
if freed > 0 {
fmt.Fprintf(out, " %s freed on the nodes\n", humanBytes(freed))
}
if v.Note != "" {
fmt.Fprintln(out, " "+v.Note)
}
return nil
},
}
c.Flags().BoolVar(&purge, "purge", false, "delete its files from the mirror and the nodes as well")
return c
}
func newModelStopCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "stop <model>",
Short: "Take a model out of service and keep it: none of its replicas run until it is started",
Long: "Its replicas stop; it stays in the catalog with its files, and a request\n" +
"for it is told an administrator stopped it. What it was scaled to is kept:\n" +
"llmgw ctl model start runs it again as it was.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var v adminapi.ScaleResponse
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/stop", nil, &v); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s stopped: its replicas stop on the next pass. It is kept; "+
"llmgw ctl model start %s runs it again.\n", v.Model, v.Model)
return nil
},
}
}
func newModelStartCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "start <model>",
Short: "Run a stopped model again, scaled as it was when it was stopped",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var v adminapi.ScaleResponse
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/start", nil, &v); err != nil {
return err
}
how := fmt.Sprintf("%d replica(s)", v.Min)
if v.OnDemand {
how = "on demand"
}
fmt.Fprintf(cmd.OutOrStdout(), "%s started: %s, max %d. Watch it with llmgw ctl models.\n", v.Model, how, v.Max)
return nil
},
}