The serve forms start at the longest context that fits
CI / test (push) Successful in 9m26s

The mirror page's form began every GGUF model at 4096 tokens, the
console's at 4096 or the plan's figure. Both now start at what the model
was trained for, cut to the most whose KV cache fits beside the weights
where the form places it; the mirror page's slots are as many of that, up
to four, as the room left holds. Plan carries the trained context.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-08 14:38:41 +01:00
1 parent 561a536477
commit 5732a5ebda
9 files changed
+152 -4

No files matched your search

+12
View File
@@ -213,10 +213,17 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
"before it is served.", modelplan.GB(p.KVBytesPerToken*1000))
}
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello."
// The longest context the first way holds, up to what the model was
// trained for: a choice of another way is checked again on submitting.
ctxDefault := "4096"
if a.Format == "safetensors" {
ctxDefault = strconv.Itoa(p.Context)
}
if len(opts) > 0 {
if most := opts[0].MostContext(p.KVBytesPerToken, p.Trained); most > 0 {
ctxDefault = strconv.Itoa(most)
}
}
if len(opts) > 0 && opts[0].Engine == "strata" {
ctxDefault = "32768"
}
@@ -249,6 +256,11 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
Description: v[3], Format: a.Format}
if a.Format == "gguf" {
req.File = a.Name
// A context longer than the default pool is the pool: one
// slot of it, where the server would refuse the request.
if ctxLen > modelserve.DefaultCtxPool {
req.CtxPool = ctxLen
}
}
o := opts[slices.Index(where, v[4])]
// Asked for a context other than planned: does it still fit?