Replicas counted per profile: seven on cards and four on CPUs under one name
A profile's replicas, set on any of a model's profiles, is how many of its replicas run there; the scaling becomes the sum. The reconciler starts the ones short of their profile before stopping the ones over, so a change of split never runs fewer. ctl model replicas sets them, adding a profile on CPUs to a GGUF model; the router already fills the higher-priority pool first, which the CPU profile is put below. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6403e61f4b
commit
e23732fbbb
13 files changed
+585
-5
No files matched your search
+49
-1
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
)
|
||||
|
||||
@@ -34,7 +35,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelStopCmd(client), newModelStartCmd(client), newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelStopCmd(client), newModelStartCmd(client), newModelReplicasCmd(client), newModelContextCmd(client), newModelGPUsCmd(client), newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -598,3 +599,50 @@ func newModelGPUsCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before its replacement starts, when there is no room beside it")
|
||||
return c
|
||||
}
|
||||
|
||||
func newModelReplicasCmd(client func() *ctlClient) *cobra.Command {
|
||||
var cpuMemory, cpuCores int
|
||||
c := &cobra.Command{
|
||||
Use: "replicas <model> <profile>=<n>...",
|
||||
Short: "How many of a model's replicas run on each of its profiles: gpu=7 cpu=4",
|
||||
Long: "Each replica runs on the best profile that fits unless counted here: seven on\n" +
|
||||
"cards and four on CPUs, all answering under the model's name, the router filling\n" +
|
||||
"the cards before the CPUs. A profile not named keeps its count. For a GGUF model\n" +
|
||||
"with no profile on CPUs, --cpu-memory adds one named cpu. Its scaling becomes the\n" +
|
||||
"sum; llmgw ctl model scale with another total ends the count per profile.",
|
||||
Example: " llmgw ctl model replicas qwen-coder gpu-1xl40s-48g=7 cpu=4 --cpu-memory 24 --cpu-cores 8",
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req := adminapi.ProfileReplicasRequest{Profiles: map[string]int{}}
|
||||
for _, a := range args[1:] {
|
||||
name, v, ok := strings.Cut(a, "=")
|
||||
n, err := strconv.Atoi(v)
|
||||
if !ok || err != nil || name == "" {
|
||||
return fmt.Errorf("%q: a profile and a count, as gpu=7", a)
|
||||
}
|
||||
req.Profiles[name] = n
|
||||
}
|
||||
if cpuMemory > 0 {
|
||||
req.CPU = &modelsvc.CPUProfile{MemoryGB: cpuMemory, CPUCores: cpuCores}
|
||||
}
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var m catalog.Model
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/replicas", body, &m); err != nil {
|
||||
return err
|
||||
}
|
||||
var parts []string
|
||||
for _, p := range m.Profiles {
|
||||
parts = append(parts, fmt.Sprintf("%d on %s", p.Replicas, p.Name))
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s runs %s: %d in all. Those short start first, then those over stop.\n",
|
||||
m.Name, strings.Join(parts, ", "), m.SplitTotal())
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().IntVar(&cpuMemory, "cpu-memory", 0, "add a profile on CPUs named cpu, reserving this many GB a replica")
|
||||
c.Flags().IntVar(&cpuCores, "cpu-cores", 0, "the cores a replica on CPUs computes on (default: half the node's)")
|
||||
return c
|
||||
}
|
||||
Reference in new issue
Block a user