From 5732a5ebda64267fe603b156f72af1d1605d961b Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 8 Nov 2026 14:38:41 +0100 Subject: [PATCH] The serve forms start at the longest context that fits The mirror page's form began every GGUF model at 4096 tokens, the console's at 4096 or the plan's figure. Both now start at what the model was trained for, cut to the most whose KV cache fits beside the weights where the form places it; the mirror page's slots are as many of that, up to four, as the room left holds. Plan carries the trained context. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++++ docs/admin-api.openapi.json | 4 ++ internal/cli/tui_hf.go | 12 ++++++ internal/control/modelserve/modelserve.go | 6 ++- internal/control/modelserve/plan.go | 5 +++ internal/modelplan/options.go | 23 +++++++++++ internal/modelplan/options_test.go | 20 +++++++++ internal/web/admin_serve.go | 49 +++++++++++++++++++++-- internal/web/admin_serve_test.go | 28 +++++++++++++ 9 files changed, 152 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 850e6a1..32e99ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,15 @@ APIs change between minors without a deprecation period. ### Fixed +- **The serve forms start at the longest context that fits.** The mirror + page's form began every GGUF model at 4096 tokens, and the console's at + 4096 or the plan's figure, whatever the model was trained for. Both now + start at what it was trained for, as long as its KV cache fits where the + form places it beside the weights, else at the most that does, in whole + thousands; the mirror page's llama.cpp slots are as many of that, up to + four, as the room left holds, and its memory figure is for that pool. A + plan says what the model was trained for (`trained`). Strata stays at + its 32768; vLLM's form is unchanged. - **`ctl weights` offered models in use for deletion.** A model of several files (a set, its address ending in `/`) is fetched into a directory named from the address without the slash; the sweep named it from the address diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 6c20685..cc16492 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -11745,6 +11745,10 @@ "context": { "type": "integer" }, + "trained": { + "type": "integer", + "description": "Trained is the context the model was trained for, the most a request may use; 0 when neither its header nor its config says." + }, "kv_bytes_per_token": { "type": "integer", "format": "int64", diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 57312bf..ab9ba21 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -213,10 +213,17 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact "before it is served.", modelplan.GB(p.KVBytesPerToken*1000)) } note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello." + // The longest context the first way holds, up to what the model was + // trained for: a choice of another way is checked again on submitting. ctxDefault := "4096" if a.Format == "safetensors" { ctxDefault = strconv.Itoa(p.Context) } + if len(opts) > 0 { + if most := opts[0].MostContext(p.KVBytesPerToken, p.Trained); most > 0 { + ctxDefault = strconv.Itoa(most) + } + } if len(opts) > 0 && opts[0].Engine == "strata" { ctxDefault = "32768" } @@ -249,6 +256,11 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact Description: v[3], Format: a.Format} if a.Format == "gguf" { req.File = a.Name + // A context longer than the default pool is the pool: one + // slot of it, where the server would refuse the request. + if ctxLen > modelserve.DefaultCtxPool { + req.CtxPool = ctxLen + } } o := opts[slices.Index(where, v[4])] // Asked for a context other than planned: does it still fit? diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 074e62f..79a57a3 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -296,7 +296,7 @@ func (s *Server) Start(req Request) (Job, error) { } req.sized = req.CtxPool > 0 || req.ContextLength > 0 if req.CtxPool <= 0 { - req.CtxPool = 16384 + req.CtxPool = DefaultCtxPool } if req.ContextLength <= 0 { req.ContextLength = 4096 @@ -731,6 +731,10 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat return out } +// DefaultCtxPool is llama.cpp's pool when a request names none: a context +// asked for longer than it needs a pool of its own. +const DefaultCtxPool = 16384 + // llamaArgs size llama.cpp's pool and its slots together. The pool is shared // by the slots, so slots of the context the catalog promises are as many as // the pool holds: the engine's default of four turned a 32768 pool into diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go index a81d797..fa2c11a 100644 --- a/internal/control/modelserve/plan.go +++ b/internal/control/modelserve/plan.go @@ -43,6 +43,9 @@ type Plan struct { License string `json:"license,omitempty"` Params int64 `json:"params,omitempty"` Context int `json:"context"` + // Trained is the context the model was trained for, the most a request + // may use; 0 when neither its header nor its config says. + Trained int `json:"trained,omitempty"` // KVBytesPerToken is one token of context, the whole model: what an // option needs for another context is worked out from it. 0 is unknown. KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"` @@ -149,6 +152,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, p.Servable, p.Reason = false, v.Reason } shape.KVBytesPerToken = v.Header.KVBytesPerToken() + p.Trained = int(v.Header.ContextLength) //nolint:gosec // a header's context length shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count shape.ActiveShare = v.Header.ActiveShare() if p.Arch == "" { @@ -167,6 +171,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, m, err := modelplan.FetchModel(ctx, repo.ID) if err == nil { shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16) + p.Trained = m.MaxContext shape.KVHeads = m.KVHeads shape.Experts = m.Experts if p.Params == 0 { diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index 2f583f1..db03b30 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -173,6 +173,29 @@ func (o Option) WithContext(kvPerToken int64, tokens int) Option { return o } +// MostContext is the longest context, up to trained, this option still holds +// for one request, in whole thousands of tokens below trained: trained when +// the KV cache's size is unknown, 0 when trained is, for Strata (its context +// is its own), or when not even 1024 tokens fit. +func (o Option) MostContext(kvPerToken int64, trained int) int { + switch { + case trained <= 0 || o.Engine == "strata": + return 0 + case kvPerToken <= 0 || o.WithContext(kvPerToken, trained).Fits(): + return trained + } + lo, hi := 0, trained/1024+1 // lo fits (or is none), hi does not + for hi-lo > 1 { + mid := (lo + hi) / 2 + if o.WithContext(kvPerToken, mid*1024).Fits() { + lo = mid + } else { + hi = mid + } + } + return lo * 1024 +} + // MemoryText says what it needs, part by part. func (o Option) MemoryText() string { if o.Engine == "strata" { diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go index 768d594..c02aa17 100644 --- a/internal/modelplan/options_test.go +++ b/internal/modelplan/options_test.go @@ -193,3 +193,23 @@ func TestAnOptionSaysWhatItNeedsOfEachCardAndOfAll(t *testing.T) { t.Errorf("a 2M-token context fits four cards: %s", huge.MemoryText()) } } + +func TestTheMostContextAnOptionHolds(t *testing.T) { + // 4 GB of weights on a 24 GB card, a token's cache 1 MiB. + o := Option{Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1, + WeightsGB: 4, EngineGB: 1, NeedGB: 5, HaveGB: 21.6} + const kv = 1 << 20 + if got := o.MostContext(kv, 8192); got != 8192 { + t.Errorf("a context that fits whole: %d, want what it was trained for", got) + } + got := o.MostContext(kv, 131072) + if got <= 0 || got%1024 != 0 || !o.WithContext(kv, got).Fits() || o.WithContext(kv, got+1024).Fits() { + t.Errorf("the most that fits is %d", got) + } + if got := o.MostContext(0, 32768); got != 32768 { + t.Errorf("an unknown cache size: %d, want what it was trained for", got) + } + if got := (Option{Engine: "strata"}).MostContext(kv, 32768); got != 0 { + t.Errorf("Strata's context is its own: %d", got) + } +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 4f4102b..6fc30eb 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -224,7 +224,10 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string, if weights <= 0 { weights = st.BytesDone } - if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 { + if c, par := mostContext(header, weights, f.Placements, f.GPUs, fleet); c > 0 { + f.Context, f.Parallel, f.CtxPool = c, par, c*par + } + if b := header.MemoryBytes(weights, f.CtxPool); b > 0 { kvHeads := header.HeadCountKV if kvHeads == 0 { kvHeads = header.HeadCount @@ -235,8 +238,8 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string, // flat "half again" could not see. f.MinMemoryGB = int((b + 1<<30 - 1) >> 30) f.MemoryBasis = l.T("serve.basis.shape", - l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30), 1), - l.Int(defaultCtxPool), header.BlockCount, kvHeads, + l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(f.CtxPool)*header.KVBytesPerToken())/(1<<30), 1), + l.Int(int64(f.CtxPool)), header.BlockCount, kvHeads, l.Num(float64(int64(1<<30)+weights/20)/(1<<30), 1)) } if fleet != nil { @@ -552,6 +555,46 @@ func modeOf(sc catalog.Scaling) string { // --parallel divides it among slots; it does not multiply it. const defaultCtxPool = 16384 +// mostContext is the longest context the serve form starts with: what the +// model was trained for, as long as its KV cache fits where the form places +// it beside the weights, else the most that does; and as many slots of it, +// up to defaultParallel, as the room left holds. 0 when the header does not +// say what it was trained for, or the weights alone do not fit. +// +// The longest, because a context is the one thing about a served model a +// client runs into without being told: a 4096 default on a model trained for +// 40960 cut documents short that the model could have read whole. +func mostContext(h gguf.Header, weights int64, places []placeChoice, gpus int, fleet Source) (tokens, slots int) { + trained := int(h.ContextLength) //nolint:gosec // a header's context length + if trained <= 0 { + return 0, 0 + } + pool := 0 + for _, pc := range places { + if !pc.Selected { + continue + } + if class, ok := strings.CutPrefix(pc.Value, "gpu:"); ok { + pool = h.PoolThatFits(weights, gguf.ClassMemGB(class), gpus) + } else { + pool = h.PoolThatFits(weights, biggestNode(fleet), 1) + } + } + switch { + case pool < 0: + return 0, 0 + case pool == 0: + // Nothing to measure it against: what it was trained for, as one slot + // would hold it; the checks on submitting still see the card. + return trained, 1 + } + tokens = min(trained, pool/1024*1024) + if tokens < 1024 { + return 0, 0 + } + return tokens, max(1, min(defaultParallel, pool/tokens)) +} + // defaultParallel is the requests a llama.cpp replica serves at once when // the form is not changed: the agent's own default. const defaultParallel = 4 diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index b9f1519..b0f5bda 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -670,3 +670,31 @@ func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) { t.Fatal("ticking the box should add it") } } + +func TestTheServeFormStartsAtTheLongestContextThatFits(t *testing.T) { + // An 8B's shape: 36 layers, 8 KV heads of 128, so 144 KiB a token. + h := gguf.Header{BlockCount: 36, HeadCount: 32, HeadCountKV: 8, KeyLength: 128, ValueLength: 128, + ContextLength: 40960} + card := []placeChoice{{Value: "gpu:6000-24g", Selected: true}, {Value: "cpu"}} + cpu := []placeChoice{{Value: "gpu:6000-24g"}, {Value: "cpu", Selected: true}} + for _, c := range []struct { + name string + weights int64 + places []placeChoice + header gguf.Header + tokens, slots int + }{ + // All it was trained for, and two slots of it in the room left. + {"on a card with room", 5 << 30, card, h, 40960, 2}, + // The weights take most of the card: the most that fits, one slot. + {"on a card nearly full", 18 << 30, card, h, 18432, 1}, + {"on the CPU of a 64 GB node", 5 << 30, cpu, h, 40960, 4}, + {"weights larger than the card", 30 << 30, card, h, 0, 0}, + {"a header that does not say", 5 << 30, card, gguf.Header{BlockCount: 36, HeadCountKV: 8, HeadCount: 32, KeyLength: 128}, 0, 0}, + } { + tokens, slots := mostContext(c.header, c.weights, c.places, 1, fakeFleet{memoryGB: 64}) + if tokens != c.tokens || slots != c.slots { + t.Errorf("%s: %d tokens × %d slots, want %d × %d", c.name, tokens, slots, c.tokens, c.slots) + } + } +}