The serve forms start at the longest context that fits

The mirror page's form began every GGUF model at 4096 tokens, the
console's at 4096 or the plan's figure. Both now start at what the model
was trained for, cut to the most whose KV cache fits beside the weights
where the form places it; the mirror page's slots are as many of that, up
to four, as the room left holds. Plan carries the trained context.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -55,6 +55,15 @@ APIs change between minors without a deprecation period.
 
 ### Fixed
 
+- **The serve forms start at the longest context that fits.** The mirror
+  page's form began every GGUF model at 4096 tokens, and the console's at
+  4096 or the plan's figure, whatever the model was trained for. Both now
+  start at what it was trained for, as long as its KV cache fits where the
+  form places it beside the weights, else at the most that does, in whole
+  thousands; the mirror page's llama.cpp slots are as many of that, up to
+  four, as the room left holds, and its memory figure is for that pool. A
+  plan says what the model was trained for (`trained`). Strata stays at
+  its 32768; vLLM's form is unchanged.
 - **`ctl weights` offered models in use for deletion.** A model of several
   files (a set, its address ending in `/`) is fetched into a directory named
   from the address without the slash; the sweep named it from the address

docs/admin-api.openapi.json

@@ -11745,6 +11745,10 @@
           "context": {
             "type": "integer"
           },
+          "trained": {
+            "type": "integer",
+            "description": "Trained is the context the model was trained for, the most a request may use; 0 when neither its header nor its config says."
+          },
           "kv_bytes_per_token": {
             "type": "integer",
             "format": "int64",

internal/cli/tui_hf.go

@@ -213,10 +213,17 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
 			"before it is served.", modelplan.GB(p.KVBytesPerToken*1000))
 	}
 	note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello."
+	// The longest context the first way holds, up to what the model was
+	// trained for: a choice of another way is checked again on submitting.
 	ctxDefault := "4096"
 	if a.Format == "safetensors" {
 		ctxDefault = strconv.Itoa(p.Context)
 	}
+	if len(opts) > 0 {
+		if most := opts[0].MostContext(p.KVBytesPerToken, p.Trained); most > 0 {
+			ctxDefault = strconv.Itoa(most)
+		}
+	}
 	if len(opts) > 0 && opts[0].Engine == "strata" {
 		ctxDefault = "32768"
 	}
@@ -249,6 +256,11 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
 				Description: v[3], Format: a.Format}
 			if a.Format == "gguf" {
 				req.File = a.Name
+				// A context longer than the default pool is the pool: one
+				// slot of it, where the server would refuse the request.
+				if ctxLen > modelserve.DefaultCtxPool {
+					req.CtxPool = ctxLen
+				}
 			}
 			o := opts[slices.Index(where, v[4])]
 			// Asked for a context other than planned: does it still fit?

internal/control/modelserve/modelserve.go

@@ -296,7 +296,7 @@ func (s *Server) Start(req Request) (Job, error) {
 	}
 	req.sized = req.CtxPool > 0 || req.ContextLength > 0
 	if req.CtxPool <= 0 {
-		req.CtxPool = 16384
+		req.CtxPool = DefaultCtxPool
 	}
 	if req.ContextLength <= 0 {
 		req.ContextLength = 4096
@@ -731,6 +731,10 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
 	return out
 }
 
+// DefaultCtxPool is llama.cpp's pool when a request names none: a context
+// asked for longer than it needs a pool of its own.
+const DefaultCtxPool = 16384
+
 // llamaArgs size llama.cpp's pool and its slots together. The pool is shared
 // by the slots, so slots of the context the catalog promises are as many as
 // the pool holds: the engine's default of four turned a 32768 pool into

internal/control/modelserve/plan.go

@@ -43,6 +43,9 @@ type Plan struct {
 	License  string `json:"license,omitempty"`
 	Params   int64  `json:"params,omitempty"`
 	Context  int    `json:"context"`
+	// Trained is the context the model was trained for, the most a request
+	// may use; 0 when neither its header nor its config says.
+	Trained int `json:"trained,omitempty"`
 	// KVBytesPerToken is one token of context, the whole model: what an
 	// option needs for another context is worked out from it. 0 is unknown.
 	KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"`
@@ -149,6 +152,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
 			p.Servable, p.Reason = false, v.Reason
 		}
 		shape.KVBytesPerToken = v.Header.KVBytesPerToken()
+		p.Trained = int(v.Header.ContextLength)   //nolint:gosec // a header's context length
 		shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count
 		shape.ActiveShare = v.Header.ActiveShare()
 		if p.Arch == "" {
@@ -167,6 +171,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
 		m, err := modelplan.FetchModel(ctx, repo.ID)
 		if err == nil {
 			shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16)
+			p.Trained = m.MaxContext
 			shape.KVHeads = m.KVHeads
 			shape.Experts = m.Experts
 			if p.Params == 0 {

internal/modelplan/options.go

@@ -173,6 +173,29 @@ func (o Option) WithContext(kvPerToken int64, tokens int) Option {
 	return o
 }
 
+// MostContext is the longest context, up to trained, this option still holds
+// for one request, in whole thousands of tokens below trained: trained when
+// the KV cache's size is unknown, 0 when trained is, for Strata (its context
+// is its own), or when not even 1024 tokens fit.
+func (o Option) MostContext(kvPerToken int64, trained int) int {
+	switch {
+	case trained <= 0 || o.Engine == "strata":
+		return 0
+	case kvPerToken <= 0 || o.WithContext(kvPerToken, trained).Fits():
+		return trained
+	}
+	lo, hi := 0, trained/1024+1 // lo fits (or is none), hi does not
+	for hi-lo > 1 {
+		mid := (lo + hi) / 2
+		if o.WithContext(kvPerToken, mid*1024).Fits() {
+			lo = mid
+		} else {
+			hi = mid
+		}
+	}
+	return lo * 1024
+}
+
 // MemoryText says what it needs, part by part.
 func (o Option) MemoryText() string {
 	if o.Engine == "strata" {

internal/modelplan/options_test.go

@@ -193,3 +193,23 @@ func TestAnOptionSaysWhatItNeedsOfEachCardAndOfAll(t *testing.T) {
 		t.Errorf("a 2M-token context fits four cards: %s", huge.MemoryText())
 	}
 }
+
+func TestTheMostContextAnOptionHolds(t *testing.T) {
+	// 4 GB of weights on a 24 GB card, a token's cache 1 MiB.
+	o := Option{Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1,
+		WeightsGB: 4, EngineGB: 1, NeedGB: 5, HaveGB: 21.6}
+	const kv = 1 << 20
+	if got := o.MostContext(kv, 8192); got != 8192 {
+		t.Errorf("a context that fits whole: %d, want what it was trained for", got)
+	}
+	got := o.MostContext(kv, 131072)
+	if got <= 0 || got%1024 != 0 || !o.WithContext(kv, got).Fits() || o.WithContext(kv, got+1024).Fits() {
+		t.Errorf("the most that fits is %d", got)
+	}
+	if got := o.MostContext(0, 32768); got != 32768 {
+		t.Errorf("an unknown cache size: %d, want what it was trained for", got)
+	}
+	if got := (Option{Engine: "strata"}).MostContext(kv, 32768); got != 0 {
+		t.Errorf("Strata's context is its own: %d", got)
+	}
+}

internal/web/admin_serve.go

@@ -224,7 +224,10 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
 	if weights <= 0 {
 		weights = st.BytesDone
 	}
-	if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 {
+	if c, par := mostContext(header, weights, f.Placements, f.GPUs, fleet); c > 0 {
+		f.Context, f.Parallel, f.CtxPool = c, par, c*par
+	}
+	if b := header.MemoryBytes(weights, f.CtxPool); b > 0 {
 		kvHeads := header.HeadCountKV
 		if kvHeads == 0 {
 			kvHeads = header.HeadCount
@@ -235,8 +238,8 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
 		// flat "half again" could not see.
 		f.MinMemoryGB = int((b + 1<<30 - 1) >> 30)
 		f.MemoryBasis = l.T("serve.basis.shape",
-			l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30), 1),
-			l.Int(defaultCtxPool), header.BlockCount, kvHeads,
+			l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(f.CtxPool)*header.KVBytesPerToken())/(1<<30), 1),
+			l.Int(int64(f.CtxPool)), header.BlockCount, kvHeads,
 			l.Num(float64(int64(1<<30)+weights/20)/(1<<30), 1))
 	}
 	if fleet != nil {
@@ -552,6 +555,46 @@ func modeOf(sc catalog.Scaling) string {
 // --parallel divides it among slots; it does not multiply it.
 const defaultCtxPool = 16384
 
+// mostContext is the longest context the serve form starts with: what the
+// model was trained for, as long as its KV cache fits where the form places
+// it beside the weights, else the most that does; and as many slots of it,
+// up to defaultParallel, as the room left holds. 0 when the header does not
+// say what it was trained for, or the weights alone do not fit.
+//
+// The longest, because a context is the one thing about a served model a
+// client runs into without being told: a 4096 default on a model trained for
+// 40960 cut documents short that the model could have read whole.
+func mostContext(h gguf.Header, weights int64, places []placeChoice, gpus int, fleet Source) (tokens, slots int) {
+	trained := int(h.ContextLength) //nolint:gosec // a header's context length
+	if trained <= 0 {
+		return 0, 0
+	}
+	pool := 0
+	for _, pc := range places {
+		if !pc.Selected {
+			continue
+		}
+		if class, ok := strings.CutPrefix(pc.Value, "gpu:"); ok {
+			pool = h.PoolThatFits(weights, gguf.ClassMemGB(class), gpus)
+		} else {
+			pool = h.PoolThatFits(weights, biggestNode(fleet), 1)
+		}
+	}
+	switch {
+	case pool < 0:
+		return 0, 0
+	case pool == 0:
+		// Nothing to measure it against: what it was trained for, as one slot
+		// would hold it; the checks on submitting still see the card.
+		return trained, 1
+	}
+	tokens = min(trained, pool/1024*1024)
+	if tokens < 1024 {
+		return 0, 0
+	}
+	return tokens, max(1, min(defaultParallel, pool/tokens))
+}
+
 // defaultParallel is the requests a llama.cpp replica serves at once when
 // the form is not changed: the agent's own default.
 const defaultParallel = 4

internal/web/admin_serve_test.go

@@ -670,3 +670,31 @@ func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) {
 		t.Fatal("ticking the box should add it")
 	}
 }
+
+func TestTheServeFormStartsAtTheLongestContextThatFits(t *testing.T) {
+	// An 8B's shape: 36 layers, 8 KV heads of 128, so 144 KiB a token.
+	h := gguf.Header{BlockCount: 36, HeadCount: 32, HeadCountKV: 8, KeyLength: 128, ValueLength: 128,
+		ContextLength: 40960}
+	card := []placeChoice{{Value: "gpu:6000-24g", Selected: true}, {Value: "cpu"}}
+	cpu := []placeChoice{{Value: "gpu:6000-24g"}, {Value: "cpu", Selected: true}}
+	for _, c := range []struct {
+		name          string
+		weights       int64
+		places        []placeChoice
+		header        gguf.Header
+		tokens, slots int
+	}{
+		// All it was trained for, and two slots of it in the room left.
+		{"on a card with room", 5 << 30, card, h, 40960, 2},
+		// The weights take most of the card: the most that fits, one slot.
+		{"on a card nearly full", 18 << 30, card, h, 18432, 1},
+		{"on the CPU of a 64 GB node", 5 << 30, cpu, h, 40960, 4},
+		{"weights larger than the card", 30 << 30, card, h, 0, 0},
+		{"a header that does not say", 5 << 30, card, gguf.Header{BlockCount: 36, HeadCountKV: 8, HeadCount: 32, KeyLength: 128}, 0, 0},
+	} {
+		tokens, slots := mostContext(c.header, c.weights, c.places, 1, fakeFleet{memoryGB: 64})
+		if tokens != c.tokens || slots != c.slots {
+			t.Errorf("%s: %d tokens × %d slots, want %d × %d", c.name, tokens, slots, c.tokens, c.slots)
+		}
+	}
+}