Replicas counted per profile: seven on cards and four on CPUs under one name

A profile's replicas, set on any of a model's profiles, is how many of its
replicas run there; the scaling becomes the sum. The reconciler starts the
ones short of their profile before stopping the ones over, so a change of
split never runs fewer. ctl model replicas sets them, adding a profile on
CPUs to a GGUF model; the router already fills the higher-priority pool
first, which the CPU profile is put below.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 09:21:50 +01:00
1 parent 6403e61f4b
commit e23732fbbb
13 files changed
+585 -5

No files matched your search

+39
View File
@@ -22,6 +22,7 @@ import (
"unicode"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
@@ -130,6 +131,7 @@ func (a *API) Register(mux Mux) {
mux.HandleFunc("POST /api/v1/models/{name}/start", a.startModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
mux.HandleFunc("POST /api/v1/models/{name}/replicas", a.setProfileReplicas)
a.registerQuotas(mux)
a.registerKB(mux)
a.registerTenancy(mux)
@@ -492,6 +494,9 @@ type ModelView struct {
Unplaceable bool `json:"unplaceable,omitempty"`
Fallbacks []string `json:"fallbacks,omitempty"`
Profiles []string `json:"profiles"`
// ProfileReplicas is how many replicas run on each profile, for a model
// whose replicas are counted per profile; absent otherwise.
ProfileReplicas map[string]int `json:"profile_replicas,omitempty"`
// MemoryGB and CPUCores are what one replica reserves, from its first
// profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
// when none is measured.
@@ -868,6 +873,12 @@ func (a *API) Models() []ModelView {
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
Role: m.Role,
}
if m.Split() {
v.ProfileReplicas = map[string]int{}
for _, p := range m.Profiles {
v.ProfileReplicas[p.Name] = p.Replicas
}
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
}
@@ -965,6 +976,15 @@ type Models interface {
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
SetProfileReplicas(ctx context.Context, name string, counts map[string]int, cpu *modelsvc.CPUProfile, by string) (catalog.Model, error)
}
// ProfileReplicasRequest counts a model's replicas per profile: profile
// name to replicas, and CPU, for a llama.cpp model with no profile on CPUs,
// the one to add.
type ProfileReplicasRequest struct {
Profiles map[string]int `json:"profiles"`
CPU *modelsvc.CPUProfile `json:"cpu,omitempty"`
}
// ResourcesRequest sets what a model's profiles reserve of a node.
@@ -1291,6 +1311,25 @@ func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, m)
}
// setProfileReplicas counts a model's replicas per profile.
func (a *API) setProfileReplicas(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req ProfileReplicasRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
m, err := a.models.SetProfileReplicas(r.Context(), r.PathValue("name"), req.Profiles, req.CPU, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, m)
}
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")