Ilker ManapandClaude Opus 5.5
78e41460f9
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
31 lines
1.1 KiB
Go
31 lines
1.1 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
func TestATargetIsSizedFromTheMeasurementAndTheRoom(t *testing.T) {
|
|
// Three 8-GPU nodes; one tp8 replica was measured at 500 tokens/s.
|
|
a, ctl := setup(t, testModels(), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
if _, err := a.Target("llama", 1000); err == nil || !strings.Contains(err.Error(), "ctl bench llama --save") {
|
|
t.Errorf("an unmeasured model should say how to measure it: %v", err)
|
|
}
|
|
ctl.Reconciler().SetMeasurements(func() map[string]reconciler.Measured {
|
|
return map[string]reconciler.Measured{"llama/tp8": {MaxConcurrency: 16, TokPerSec: 500}}
|
|
})
|
|
v, err := a.Target("llama", 1200)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v.Needed != 3 || v.PerReplica != 500 || v.Profile != "tp8" || v.More != 3 || !v.Fits {
|
|
t.Errorf("1200 tok/s at 500 each is 3 replicas on 3 empty nodes: %+v", v)
|
|
}
|
|
v, _ = a.Target("llama", 2000)
|
|
if v.Needed != 4 || v.Fits || v.Room != 3 || len(v.Blocked) == 0 {
|
|
t.Errorf("a fourth does not fit on three nodes, and should say why: %+v", v)
|
|
}
|
|
}
|