The mirror page's form began every GGUF model at 4096 tokens, the console's at 4096 or the plan's figure. Both now start at what the model was trained for, cut to the most whose KV cache fits beside the weights where the form places it; the mirror page's slots are as many of that, up to four, as the room left holds. Plan carries the trained context. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
561a536477
commit
5732a5ebda
9 files changed
+152
-4
No files matched your search
@@ -213,10 +213,17 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
|
||||
"before it is served.", modelplan.GB(p.KVBytesPerToken*1000))
|
||||
}
|
||||
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello."
|
||||
// The longest context the first way holds, up to what the model was
|
||||
// trained for: a choice of another way is checked again on submitting.
|
||||
ctxDefault := "4096"
|
||||
if a.Format == "safetensors" {
|
||||
ctxDefault = strconv.Itoa(p.Context)
|
||||
}
|
||||
if len(opts) > 0 {
|
||||
if most := opts[0].MostContext(p.KVBytesPerToken, p.Trained); most > 0 {
|
||||
ctxDefault = strconv.Itoa(most)
|
||||
}
|
||||
}
|
||||
if len(opts) > 0 && opts[0].Engine == "strata" {
|
||||
ctxDefault = "32768"
|
||||
}
|
||||
@@ -249,6 +256,11 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
|
||||
Description: v[3], Format: a.Format}
|
||||
if a.Format == "gguf" {
|
||||
req.File = a.Name
|
||||
// A context longer than the default pool is the pool: one
|
||||
// slot of it, where the server would refuse the request.
|
||||
if ctxLen > modelserve.DefaultCtxPool {
|
||||
req.CtxPool = ctxLen
|
||||
}
|
||||
}
|
||||
o := opts[slices.Index(where, v[4])]
|
||||
// Asked for a context other than planned: does it still fit?
|
||||
|
||||
Reference in new issue
Block a user