Scaling by the clock: schedule windows with their own min and max, prewarm, control.timezone

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 23:16:56 +02:00
1 parent 74429f1c00
commit bce54f0941
14 files changed
+382 -9

No files matched your search

+10
View File
@@ -64,6 +64,7 @@ var Reference = []Block{
{Key: "lost_after", Type: "time.Duration", Doc: "", Nested: ""},
{Key: "port_range", Type: "[]int", Doc: "PortRange is the per-node range for engine HTTP and dist-init ports.", Nested: ""},
{Key: "mgmt_interface", Type: "string", Doc: "MgmtInterface is the ethernet interface NCCL bootstraps over.", Nested: ""},
{Key: "timezone", Type: "string", Doc: "Timezone is where models' scaling schedules are read, an IANA name\nlike Europe/Istanbul; empty is the control plane's own.", Nested: ""},
{Key: "enabled", Type: "bool", Doc: "Enabled runs the control plane even when configuration declares no\nmodels.\n\nWithout it, a deployment whose catalog lives entirely in the database\ncannot start: control plane mode is inferred from the config file, so an\nempty models block means \"router only\", the process refuses to start for\nwant of anything to serve, and the models it would have loaded are never\nread. That is a cluster that cannot be managed at runtime because it is\nmanaged at runtime. AgentListen cannot carry this signal — it has a\ndefault, so every deployment would become a control plane.", Nested: ""},
{Key: "lease_ttl", Type: "time.Duration", Doc: "LeaseTTL is how long this control plane's claim to be the one placing\nwork stands without being renewed.\n\nOnly meaningful with a shared store, where a second control plane can\nsee the claim. Long enough to survive a slow query or a brief network\nstall, short enough that a standby takes over before anyone has finished\nreading the alert. Zero takes the default.", Nested: ""},
}},
@@ -268,10 +269,19 @@ var Reference = []Block{
{Key: "ttft_p99", Type: "time.Duration", Doc: "TTFTp99 is the time to the first token 99 requests in 100 are within.", Nested: ""},
{Key: "tpot_p99", Type: "time.Duration", Doc: "TPOTp99 is the time per output token, after the first, 99 streamed\nrequests in 100 are within.", Nested: ""},
}},
{Package: "catalog", Name: "ScaleWindow", Title: "ScaleWindow", Doc: "", Fields: []Field{
{Key: "days", Type: "[]string", Doc: "Days are mon … sun, or weekdays, weekends; empty is every day.", Nested: ""},
{Key: "from", Type: "string", Doc: "From and To are HH:MM; a window whose To is before its From runs\npast midnight.", Nested: ""},
{Key: "to", Type: "string", Doc: "", Nested: ""},
{Key: "min", Type: "int", Doc: "", Nested: ""},
{Key: "max", Type: "int", Doc: "Max, when set, is the window's ceiling for load scaling; zero keeps\nthe model's own.", Nested: ""},
}},
{Package: "catalog", Name: "Scaling", Title: "Scaling", Doc: "", Fields: []Field{
{Key: "min", Type: "int", Doc: "", Nested: ""},
{Key: "max", Type: "int", Doc: "", Nested: ""},
{Key: "idle_timeout", Type: "time.Duration", Doc: "IdleTimeout is how long an on-demand model may go unasked-for before its\nreplicas are stopped. Zero takes the cluster default.\n\nMinutes rather than seconds, always. Stopping a replica throws away the\nKV cache prefix routing spent its effort filling, and a model that flaps\nin and out loses that entirely — so the saving has to be worth more than\nthe cache, which it only is over a real idle period.", Nested: ""},
{Key: "schedule", Type: "[]ScaleWindow", Doc: "Schedule changes min and max by the time of day, in the control\nplane's control.timezone: more replicas kept through working hours,\nfewer at night, an on-demand model kept loaded while people are at\nwork. The first window that holds now wins; outside them all, Min and\nMax apply.", Nested: ""},
{Key: "prewarm", Type: "time.Duration", Doc: "Prewarm starts a window this long before it opens, so its replicas\nhave loaded by then: loading a model is minutes, and the people who\narrive at nine are the reason for the window.", Nested: ""},
}},
{Package: "config", Name: "Scoring", Title: "Scoring", Doc: "", Fields: []Field{
{Key: "target_waiting", Type: "float64", Doc: "", Nested: ""},