Replicas counted per profile: seven on cards and four on CPUs under one name

A profile's replicas, set on any of a model's profiles, is how many of its
replicas run there; the scaling becomes the sum. The reconciler starts the
ones short of their profile before stopping the ones over, so a change of
split never runs fewer. ctl model replicas sets them, adding a profile on
CPUs to a GGUF model; the router already fills the higher-priority pool
first, which the CPU profile is put below.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 09:21:50 +01:00
1 parent 6403e61f4b
commit e23732fbbb
13 files changed
+585 -5

No files matched your search

+49 -1
View File
@@ -19,6 +19,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
)
@@ -34,7 +35,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelStopCmd(client), newModelStartCmd(client), newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
c.AddCommand(newModelStopCmd(client), newModelStartCmd(client), newModelReplicasCmd(client), newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
newTargetCmd(client, asJSON), newAdaptersCmd(client),
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
@@ -598,3 +599,50 @@ func newModelGPUsCmd(client func() *ctlClient) *cobra.Command {
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before its replacement starts, when there is no room beside it")
return c
}
func newModelReplicasCmd(client func() *ctlClient) *cobra.Command {
var cpuMemory, cpuCores int
c := &cobra.Command{
Use: "replicas <model> <profile>=<n>...",
Short: "How many of a model's replicas run on each of its profiles: gpu=7 cpu=4",
Long: "Each replica runs on the best profile that fits unless counted here: seven on\n" +
"cards and four on CPUs, all answering under the model's name, the router filling\n" +
"the cards before the CPUs. A profile not named keeps its count. For a GGUF model\n" +
"with no profile on CPUs, --cpu-memory adds one named cpu. Its scaling becomes the\n" +
"sum; llmgw ctl model scale with another total ends the count per profile.",
Example: " llmgw ctl model replicas qwen-coder gpu-1xl40s-48g=7 cpu=4 --cpu-memory 24 --cpu-cores 8",
Args: cobra.MinimumNArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
req := adminapi.ProfileReplicasRequest{Profiles: map[string]int{}}
for _, a := range args[1:] {
name, v, ok := strings.Cut(a, "=")
n, err := strconv.Atoi(v)
if !ok || err != nil || name == "" {
return fmt.Errorf("%q: a profile and a count, as gpu=7", a)
}
req.Profiles[name] = n
}
if cpuMemory > 0 {
req.CPU = &modelsvc.CPUProfile{MemoryGB: cpuMemory, CPUCores: cpuCores}
}
body, err := json.Marshal(req)
if err != nil {
return err
}
var m catalog.Model
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/replicas", body, &m); err != nil {
return err
}
var parts []string
for _, p := range m.Profiles {
parts = append(parts, fmt.Sprintf("%d on %s", p.Replicas, p.Name))
}
fmt.Fprintf(cmd.OutOrStdout(), "%s runs %s: %d in all. Those short start first, then those over stop.\n",
m.Name, strings.Join(parts, ", "), m.SplitTotal())
return nil
},
}
c.Flags().IntVar(&cpuMemory, "cpu-memory", 0, "add a profile on CPUs named cpu, reserving this many GB a replica")
c.Flags().IntVar(&cpuCores, "cpu-cores", 0, "the cores a replica on CPUs computes on (default: half the node's)")
return c
}