Files
llmgw/internal/cli/model.go
T
Ilker ManapandClaude Opus 5.5 990d82d801
Release / build (push) Successful in 1m5s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m47s
Reserve CPU cores and require memory for CPU models
A profile's cpu_cores is reserved out of its node's CPUs by placement, as
memory is, and becomes the engine's thread count. Agents report their CPU
count in registration and heartbeats. New CPU models must declare memory;
model views show reserved against measured memory with warnings, and
ctl model resources sets both for a stored model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 17:43:13 +02:00

236 lines
9.3 KiB
Go

package cli
import (
"encoding/json"
"fmt"
"os"
"os/user"
"time"
"github.com/knadh/koanf/parsers/yaml"
"github.com/knadh/koanf/providers/file"
"github.com/knadh/koanf/v2"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
)
// newModelCmd adds and removes models without a restart.
//
// Adding a model used to mean editing a file on the control plane and
// restarting it: a settling window, adoption churn, and on one occasion two
// minutes of outage. For a fixed production set that is a reasonable trade; for
// a platform whose promise is that people can try things, it is the difference
// between trying one and not bothering.
func newModelCmd(client func() *ctlClient) *cobra.Command {
c := &cobra.Command{
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
newModelServeCmd(client))
return c
}
func newModelAddCmd(client func() *ctlClient) *cobra.Command {
var path string
var skipCheck bool
c := &cobra.Command{
Use: "add -f <file.yaml>",
Short: "Add a model from a definition file",
Long: "The definition is the same YAML a model takes in server.yaml, so an\n" +
"entry can be moved between the two by copying it.\n\n" +
"The catalog is rebuilt and validated before anything is written, so a\n" +
"definition that collides with an existing name, or names a fallback that\n" +
"does not exist, is refused rather than stored and then ignored on every\n" +
"start. Once accepted it takes effect on the reconciler's next pass — no\n" +
"restart — and survives one.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if path == "" {
return fmt.Errorf("--file is required: the model definition, as YAML")
}
// Read by koanf, the same parser and the same tags that read
// server.yaml — so an entry can be moved between the two by copying
// it, and a field that works in one cannot silently mean nothing in
// the other.
k := koanf.New(".")
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
var m catalog.Model
if err := k.UnmarshalWithConf("", &m, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
if m.Name == "" {
return fmt.Errorf("%s: the definition has no name", path)
}
// Before the catalog is touched, because a source that does not
// resolve is otherwise discovered by a replica failing to start
// minutes later on a node — with an error naming the node rather
// than the typo.
if !skipCheck {
size, err := sourcecheck.Check(cmd.Context(), m.Source)
if err != nil {
return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+
"which is right when the mirror is not up yet or the nodes "+
"reach it by an address this machine does not.", err)
}
if _, isURL := sourcecheck.HTTPSource(m.Source); isURL {
fmt.Fprintln(cmd.OutOrStdout(), sourceNote(m.Source, size))
}
}
body, err := json.Marshal(m)
if err != nil {
return err
}
var res map[string]string
if err := client().post(cmd.Context(), "/api/v1/models", body, &res); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(),
"%s added. It will be placed on the next reconcile pass; watch it with:\n"+
" llmgw ctl models\n", m.Name)
return nil
},
}
c.Flags().StringVarP(&path, "file", "f", "", "YAML file holding the model definition")
c.Flags().BoolVar(&skipCheck, "no-source-check", false,
"add it without asking whether the weights are actually at that URL")
return c
}
func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "remove <name>",
Short: "Remove a model that was added at runtime",
Long: "Only models added at runtime. One declared in server.yaml would come\n" +
"back on the next restart, so removing it here would leave the cluster\n" +
"disagreeing with its own configuration until then.\n\n" +
"Its replicas are stopped on the next pass, after the endpoint has been\n" +
"withdrawn, so traffic stops before the engine does.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var res map[string]string
if err := client().delete(cmd.Context(), "/api/v1/models/"+args[0], &res); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s removed.\n", args[0])
return nil
},
}
}
// actor names who made a change, for the audit trail on a stored model.
func actor() string {
// Under sudo, the person who ran it rather than root.
if u := os.Getenv("SUDO_USER"); u != "" && os.Geteuid() == 0 {
return u
}
if u, err := user.Current(); err == nil && u.Username != "" {
return u.Username
}
return "unknown"
}
func newScaleCmd(client func() *ctlClient) *cobra.Command {
var min, max int
var idle time.Duration
c := &cobra.Command{
Use: "scale <model> --min N [--max M]",
Short: "Change how many replicas a model runs",
Long: "The replica count is --min. --max is a ceiling, not a target: a model\n" +
"with a minimum runs exactly that many, and nothing scales it up under\n" +
"load. Leaving --max out makes it follow --min.\n\n" +
"--min 0 with a --max above it is the on-demand model: nothing runs until\n" +
"somebody asks, and it stops again after the idle timeout. That loads one\n" +
"replica however high --max is. --min 0 --max 0 is a model kept in the\n" +
"catalog and deliberately not running.\n\n" +
"Only models added at runtime. One declared in server.yaml is read again\n" +
"on every start, so a number set here would hold until the next restart\n" +
"and then revert — change it in the file instead.\n\n" +
"It takes effect on the reconciler's next pass, and survives a restart.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if !cmd.Flags().Changed("min") {
return fmt.Errorf("--min is required: it is the replica count")
}
body, err := json.Marshal(adminapi.ScaleRequest{Min: min, Max: max, IdleTimeout: idle})
if err != nil {
return err
}
var res adminapi.ScaleResponse
if err := client().post(cmd.Context(),
"/api/v1/models/"+args[0]+"/scale", body, &res); err != nil {
return err
}
// What is in force, not what was asked for: the catalog fills in a
// max that was left out, and printing the request back would show a
// number the cluster is not using.
out := cmd.OutOrStdout()
switch {
case res.OnDemand:
fmt.Fprintf(out, "%s is on-demand: loaded when asked for, up to %d, stopped when idle.\n",
res.Model, res.Max)
case res.Min == 0:
fmt.Fprintf(out, "%s will not be placed; it stays in the catalog.\n", res.Model)
default:
fmt.Fprintf(out, "%s: %d replicas (max %d). It takes effect on the next pass; watch it with:\n"+
" llmgw ctl models\n", res.Model, res.Min, res.Max)
}
return nil
},
}
c.Flags().IntVar(&min, "min", 0, "replicas to keep running; this is the count")
c.Flags().IntVar(&max, "max", 0, "ceiling for an on-demand model; follows --min when left out")
c.Flags().DurationVar(&idle, "idle-timeout", 0,
"how long an on-demand model stays up unasked-for; zero takes the cluster default")
return c
}
func newResourcesCmd(client func() *ctlClient) *cobra.Command {
var memory, cores int
var profile string
c := &cobra.Command{
Use: "resources <model> --memory GB [--cores N]",
Short: "Set what a model reserves of a node: memory, and CPU cores",
Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" +
"weights, KV cache and engine together. Placement reserves it on the node,\n" +
"so a node holds only as many replicas as it has memory for.\n\n" +
"--cores is the CPU cores a replica with no GPU computes on. They are\n" +
"reserved the same way, and become the engine's thread count, so two\n" +
"replicas on one node stop competing for the same cores. Zero reserves\n" +
"none and leaves the thread count to the node.\n\n" +
"Running replicas keep what they were started with; `llmgw ctl restart\n" +
"--model M` moves them to the new figures.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if !cmd.Flags().Changed("memory") {
return fmt.Errorf("--memory is required")
}
body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores})
if err != nil {
return err
}
var m catalog.Model
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil {
return err
}
for _, p := range m.Profiles {
fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores)
}
fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name)
return nil
},
}
c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB")
c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none")
c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
return c
}