Change a model's replica count without editing a file on the control plane
The count is scaling.min, and until now the only way to change it was to edit server.yaml on the host and restart — or, for a model added at runtime, to remove it and add it back, which means having its whole definition to hand. `llmgw ctl scale <model> --min N` instead. Only the scaling block moves: everything else is read back from what was stored and written out untouched, so changing a number cannot quietly become a redefinition. The catalog is rebuilt and validated before anything is written, as in Add. A max below a min is refused with the stored definition untouched — persisting a rejected change would come back on the next restart as a catalog that does not build. The answer reports the scaling in force rather than the request. The catalog fills in a max that was left out, and a caller echoing its own request would print a number the cluster is not using. A model declared in configuration is refused, for the reason Remove already refuses one: the YAML is read on every start, so the new count would hold until the next restart and then silently revert. A cluster disagreeing with its own config file is worse than an error saying where to edit. The help says what --max does, because it is not what it looks like: a model with a minimum runs exactly that many and nothing scales it up under load, and an on-demand model loads one replica however high its max. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
10e323132c
commit
707d774c76
6 files changed
+388
No files matched your search
@@ -4,12 +4,14 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/user"
|
||||
"time"
|
||||
|
||||
"github.com/knadh/koanf/parsers/yaml"
|
||||
"github.com/knadh/koanf/providers/file"
|
||||
"github.com/knadh/koanf/v2"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
@@ -128,3 +130,58 @@ func actor() string {
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func newScaleCmd(client func() *ctlClient) *cobra.Command {
|
||||
var min, max int
|
||||
var idle time.Duration
|
||||
c := &cobra.Command{
|
||||
Use: "scale <model> --min N [--max M]",
|
||||
Short: "Change how many replicas a model runs",
|
||||
Long: "The replica count is --min. --max is a ceiling, not a target: a model\n" +
|
||||
"with a minimum runs exactly that many, and nothing scales it up under\n" +
|
||||
"load. Leaving --max out makes it follow --min.\n\n" +
|
||||
"--min 0 with a --max above it is the on-demand model: nothing runs until\n" +
|
||||
"somebody asks, and it stops again after the idle timeout. That loads one\n" +
|
||||
"replica however high --max is. --min 0 --max 0 is a model kept in the\n" +
|
||||
"catalog and deliberately not running.\n\n" +
|
||||
"Only models added at runtime. One declared in server.yaml is read again\n" +
|
||||
"on every start, so a number set here would hold until the next restart\n" +
|
||||
"and then revert — change it in the file instead.\n\n" +
|
||||
"It takes effect on the reconciler's next pass, and survives a restart.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if !cmd.Flags().Changed("min") {
|
||||
return fmt.Errorf("--min is required: it is the replica count")
|
||||
}
|
||||
body, err := json.Marshal(adminapi.ScaleRequest{Min: min, Max: max, IdleTimeout: idle})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var res adminapi.ScaleResponse
|
||||
if err := client().post(cmd.Context(),
|
||||
"/api/v1/models/"+args[0]+"/scale", body, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
// What is in force, not what was asked for: the catalog fills in a
|
||||
// max that was left out, and printing the request back would show a
|
||||
// number the cluster is not using.
|
||||
out := cmd.OutOrStdout()
|
||||
switch {
|
||||
case res.OnDemand:
|
||||
fmt.Fprintf(out, "%s is on-demand: loaded when asked for, up to %d, stopped when idle.\n",
|
||||
res.Model, res.Max)
|
||||
case res.Min == 0:
|
||||
fmt.Fprintf(out, "%s will not be placed; it stays in the catalog.\n", res.Model)
|
||||
default:
|
||||
fmt.Fprintf(out, "%s: %d replicas (max %d). It takes effect on the next pass; watch it with:\n"+
|
||||
" llmgw ctl models\n", res.Model, res.Min, res.Max)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().IntVar(&min, "min", 0, "replicas to keep running; this is the count")
|
||||
c.Flags().IntVar(&max, "max", 0, "ceiling for an on-demand model; follows --min when left out")
|
||||
c.Flags().DurationVar(&idle, "idle-timeout", 0,
|
||||
"how long an on-demand model stays up unasked-for; zero takes the cluster default")
|
||||
return c
|
||||
}
|
||||
Reference in new issue
Block a user