CHANGELOG: one list
CI / test (push) Successful in 9m18s

This commit is contained in:
Ilker Manap committed 2026-10-30 10:39:30 +01:00
1 parent 7465c24830
commit d6f8f433f4
24 files changed
+437 -22

No files matched your search

+2
View File
@@ -64,6 +64,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
"--embedding, 32 slots of 1024 tokens unless --ctx-pool or --context-length says otherwise")
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says")
f.StringVar(&req.Engine, "engine", "", "strata, for a family Strata runs: one card and the experts in RAM; "+
"llamacpp to keep it off Strata (default: the plan's best)")
f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says")
f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)")
f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)")
+7
View File
@@ -310,6 +310,9 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
if b.CPU {
holds, all = fmt.Sprintf("%.1f GB of RAM", b.NeedGB), "-"
}
if b.Engine == "strata" {
holds, all = fmt.Sprintf("%.0f GB of RAM", b.NeedGB), "1 card"
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, all, now)
}
_ = w.Flush()
@@ -339,6 +342,10 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
if p.Format == "safetensors" {
fmt.Fprint(out, " --format safetensors")
}
if rec.Engine == "strata" {
fmt.Fprintf(out, " --file %s --engine strata\n", rec.Artifact)
return nil
}
fmt.Fprintln(out, " --fallbacks")
return nil
}
+12 -3
View File
@@ -206,6 +206,9 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
if a.Format == "safetensors" {
ctxDefault = strconv.Itoa(p.Context)
}
if len(opts) > 0 && opts[0].Engine == "strata" {
ctxDefault = "32768"
}
fields := []formField{
textF("name", hf.ModelName(p.Repo), "what clients ask for"),
textF("replicas", "1", "0 runs none until it is first asked for"),
@@ -241,12 +244,18 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
if sized := o.WithContext(p.KVBytesPerToken, ctxLen); !sized.Fits() {
return nil, fmt.Errorf("with a %d-token context it needs %s: a shorter context, or another way", ctxLen, sized.MemoryText())
}
if o.CPU {
switch {
case o.Engine == "strata":
// One card, the experts in RAM; Strata reads the pictures
// with a reader of its own, not this projector.
req.Engine, req.GPUClass, proj = "strata", o.Class, -1
case o.CPU:
req.CPU = true
} else {
default:
req.Engine = "llamacpp"
req.GPUClass, req.GPUsPerNode, req.Nodes = o.Class, o.GPUsPerNode, max(o.Nodes, 1)
}
req.Fallbacks = strings.HasPrefix(v[5], "the next") && len(opts) > 1
req.Fallbacks = strings.HasPrefix(v[5], "the next") && len(opts) > 1 && o.Engine != "strata"
if proj >= 0 && !strings.HasPrefix(v[proj], "none") {
req.Projector = v[proj]
}