Scale and place by what a replica was measured to serve
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9dc1b62fdc
commit
78e41460f9
16 files changed
+503
-15
No files matched your search
@@ -24,13 +24,14 @@ import (
|
||||
// minutes of outage. For a fixed production set that is a reasonable trade; for
|
||||
// a platform whose promise is that people can try things, it is the difference
|
||||
// between trying one and not bothering.
|
||||
func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||
func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON),
|
||||
newModelServeCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user