Replicas counted per profile: seven on cards and four on CPUs under one name
A profile's replicas, set on any of a model's profiles, is how many of its replicas run there; the scaling becomes the sum. The reconciler starts the ones short of their profile before stopping the ones over, so a change of split never runs fewer. ctl model replicas sets them, adding a profile on CPUs to a GGUF model; the router already fills the higher-priority pool first, which the CPU profile is put below. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6403e61f4b
commit
e23732fbbb
13 files changed
+585
-5
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
"unicode"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
@@ -130,6 +131,7 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/start", a.startModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/replicas", a.setProfileReplicas)
|
||||
a.registerQuotas(mux)
|
||||
a.registerKB(mux)
|
||||
a.registerTenancy(mux)
|
||||
@@ -492,6 +494,9 @@ type ModelView struct {
|
||||
Unplaceable bool `json:"unplaceable,omitempty"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||
Profiles []string `json:"profiles"`
|
||||
// ProfileReplicas is how many replicas run on each profile, for a model
|
||||
// whose replicas are counted per profile; absent otherwise.
|
||||
ProfileReplicas map[string]int `json:"profile_replicas,omitempty"`
|
||||
// MemoryGB and CPUCores are what one replica reserves, from its first
|
||||
// profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
|
||||
// when none is measured.
|
||||
@@ -868,6 +873,12 @@ func (a *API) Models() []ModelView {
|
||||
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
|
||||
Role: m.Role,
|
||||
}
|
||||
if m.Split() {
|
||||
v.ProfileReplicas = map[string]int{}
|
||||
for _, p := range m.Profiles {
|
||||
v.ProfileReplicas[p.Name] = p.Replicas
|
||||
}
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
}
|
||||
@@ -965,6 +976,15 @@ type Models interface {
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
|
||||
SetProfileReplicas(ctx context.Context, name string, counts map[string]int, cpu *modelsvc.CPUProfile, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ProfileReplicasRequest counts a model's replicas per profile: profile
|
||||
// name to replicas, and CPU, for a llama.cpp model with no profile on CPUs,
|
||||
// the one to add.
|
||||
type ProfileReplicasRequest struct {
|
||||
Profiles map[string]int `json:"profiles"`
|
||||
CPU *modelsvc.CPUProfile `json:"cpu,omitempty"`
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
@@ -1291,6 +1311,25 @@ func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, m)
|
||||
}
|
||||
|
||||
// setProfileReplicas counts a model's replicas per profile.
|
||||
func (a *API) setProfileReplicas(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req ProfileReplicasRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
m, err := a.models.SetProfileReplicas(r.Context(), r.PathValue("name"), req.Profiles, req.CPU, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, m)
|
||||
}
|
||||
|
||||
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
|
||||
Reference in new issue
Block a user