Replicas counted per profile: seven on cards and four on CPUs under one name
A profile's replicas, set on any of a model's profiles, is how many of its replicas run there; the scaling becomes the sum. The reconciler starts the ones short of their profile before stopping the ones over, so a change of split never runs fewer. ctl model replicas sets them, adding a profile on CPUs to a GGUF model; the router already fills the higher-priority pool first, which the CPU profile is put below. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6403e61f4b
commit
e23732fbbb
13 files changed
+585
-5
No files matched your search
@@ -267,6 +267,12 @@ type Profile struct {
|
||||
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
||||
// Priority orders profiles during placement; highest that fits wins.
|
||||
Priority int `json:"priority,omitempty" koanf:"priority"`
|
||||
// Replicas, set on any of a model's profiles, is how many of its
|
||||
// replicas run on this one: seven on cards and four on CPUs, all
|
||||
// answering under the model's name, rather than each replica on the
|
||||
// best profile that fits. The model's scaling is then the sum, min and
|
||||
// max alike; a profile left at zero runs none.
|
||||
Replicas int `json:"replicas,omitempty" koanf:"replicas"`
|
||||
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
||||
// tens of minutes, so this is per profile rather than global.
|
||||
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
||||
@@ -1024,10 +1030,57 @@ func (m Model) normalize() Model {
|
||||
return m
|
||||
}
|
||||
|
||||
// Split says whether the model's replicas are counted per profile: any of
|
||||
// its profiles sets replicas.
|
||||
func (m Model) Split() bool {
|
||||
return slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Replicas > 0 })
|
||||
}
|
||||
|
||||
// SplitTotal is the replicas its profiles ask for together.
|
||||
func (m Model) SplitTotal() int {
|
||||
n := 0
|
||||
for _, p := range m.Profiles {
|
||||
n += max(p.Replicas, 0)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ProfileBase is the catalog profile a placement profile stands for: one
|
||||
// named by the card it needs is placed as name@class.
|
||||
func ProfileBase(name string) string {
|
||||
base, _, _ := strings.Cut(name, "@")
|
||||
return base
|
||||
}
|
||||
|
||||
// validateSplit refuses replicas per profile that the model's scaling does
|
||||
// not add up to.
|
||||
func (m Model) validateSplit() error {
|
||||
for _, p := range m.Profiles {
|
||||
if p.Replicas < 0 {
|
||||
return fmt.Errorf("model %q profile %q: replicas %d is negative", m.Name, p.Name, p.Replicas)
|
||||
}
|
||||
}
|
||||
if !m.Split() || m.Stopped != nil {
|
||||
return nil
|
||||
}
|
||||
n := m.SplitTotal()
|
||||
if m.Scaling.Min != n || m.Scaling.Max != n {
|
||||
return fmt.Errorf("model %q: its profiles' replicas add up to %d, so its scaling is min %d, max %d, not %d and %d",
|
||||
m.Name, n, n, n, m.Scaling.Min, m.Scaling.Max)
|
||||
}
|
||||
if len(m.Scaling.Schedule) > 0 {
|
||||
return fmt.Errorf("model %q: replicas per profile and a schedule are one or the other", m.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Model) validate() error {
|
||||
if m.Name == "" {
|
||||
return fmt.Errorf("a model has no name")
|
||||
}
|
||||
if err := m.validateSplit(); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Source == "" {
|
||||
return fmt.Errorf("model %q: source is required", m.Name)
|
||||
}
|
||||
@@ -1510,6 +1563,9 @@ func (m Model) Simulated() bool {
|
||||
// happened, and the answer to "why is this one on the wrong node" was in
|
||||
// somebody's memory of what they edited an hour ago.
|
||||
func (p Profile) Fingerprint() string {
|
||||
// How many replicas run on it is not how each runs: changing the count
|
||||
// must not mark the running ones stale.
|
||||
p.Replicas = 0
|
||||
b, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
// A profile that will not marshal cannot be compared, and saying so by
|
||||
|
||||
Reference in new issue
Block a user