Replicas counted per profile: seven on cards and four on CPUs under one name

A profile's replicas, set on any of a model's profiles, is how many of its
replicas run there; the scaling becomes the sum. The reconciler starts the
ones short of their profile before stopping the ones over, so a change of
split never runs fewer. ctl model replicas sets them, adding a profile on
CPUs to a GGUF model; the router already fills the higher-priority pool
first, which the CPU profile is put below.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 09:21:50 +01:00
1 parent 6403e61f4b
commit e23732fbbb
13 files changed
+585 -5

No files matched your search

+56
View File
@@ -267,6 +267,12 @@ type Profile struct {
Weight float64 `json:"weight,omitempty" koanf:"weight"`
// Priority orders profiles during placement; highest that fits wins.
Priority int `json:"priority,omitempty" koanf:"priority"`
// Replicas, set on any of a model's profiles, is how many of its
// replicas run on this one: seven on cards and four on CPUs, all
// answering under the model's name, rather than each replica on the
// best profile that fits. The model's scaling is then the sum, min and
// max alike; a profile left at zero runs none.
Replicas int `json:"replicas,omitempty" koanf:"replicas"`
// StartupTimeout bounds model loading. A 70B model across two nodes takes
// tens of minutes, so this is per profile rather than global.
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
@@ -1024,10 +1030,57 @@ func (m Model) normalize() Model {
return m
}
// Split says whether the model's replicas are counted per profile: any of
// its profiles sets replicas.
func (m Model) Split() bool {
return slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Replicas > 0 })
}
// SplitTotal is the replicas its profiles ask for together.
func (m Model) SplitTotal() int {
n := 0
for _, p := range m.Profiles {
n += max(p.Replicas, 0)
}
return n
}
// ProfileBase is the catalog profile a placement profile stands for: one
// named by the card it needs is placed as name@class.
func ProfileBase(name string) string {
base, _, _ := strings.Cut(name, "@")
return base
}
// validateSplit refuses replicas per profile that the model's scaling does
// not add up to.
func (m Model) validateSplit() error {
for _, p := range m.Profiles {
if p.Replicas < 0 {
return fmt.Errorf("model %q profile %q: replicas %d is negative", m.Name, p.Name, p.Replicas)
}
}
if !m.Split() || m.Stopped != nil {
return nil
}
n := m.SplitTotal()
if m.Scaling.Min != n || m.Scaling.Max != n {
return fmt.Errorf("model %q: its profiles' replicas add up to %d, so its scaling is min %d, max %d, not %d and %d",
m.Name, n, n, n, m.Scaling.Min, m.Scaling.Max)
}
if len(m.Scaling.Schedule) > 0 {
return fmt.Errorf("model %q: replicas per profile and a schedule are one or the other", m.Name)
}
return nil
}
func (m Model) validate() error {
if m.Name == "" {
return fmt.Errorf("a model has no name")
}
if err := m.validateSplit(); err != nil {
return err
}
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
@@ -1510,6 +1563,9 @@ func (m Model) Simulated() bool {
// happened, and the answer to "why is this one on the wrong node" was in
// somebody's memory of what they edited an hour ago.
func (p Profile) Fingerprint() string {
// How many replicas run on it is not how each runs: changing the count
// must not mark the running ones stale.
p.Replicas = 0
b, err := json.Marshal(p)
if err != nil {
// A profile that will not marshal cannot be compared, and saying so by