CHANGELOG: one list
CI / test (push) Successful in 9m18s

This commit is contained in:
Ilker Manap committed 2026-10-30 10:39:30 +01:00
1 parent 7465c24830
commit d6f8f433f4
24 files changed
+437 -22

No files matched your search

+5 -1
View File
@@ -823,7 +823,11 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
Version: got.ShortRevision(),
}
rs.Verdict = p.verdictFor(langOf(r), got.Architecture, set.Name)
if fit != nil && rs.Verdict.Servable {
// A family Strata runs: one card and the experts in RAM, whether
// or not llama.cpp could load it.
if layout, ok := fit.strataLayout(langOf(r), got.ID, set.Name, set.Bytes); ok {
rs.Verdict, rs.GPULayout = gguf.Verdict{Checked: true, Servable: true}, layout
} else if fit != nil && rs.Verdict.Servable {
if v, ok := fit.holds(langOf(r), set.Name, set.Bytes, got.Params); !ok {
rs.Verdict = v
} else if gguf.IsGGUF(set.Name) {
+61 -2
View File
@@ -4,14 +4,18 @@ import (
"context"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"slices"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
)
// CatalogAdmin adds a model to the catalogue in force.
@@ -76,6 +80,11 @@ type serveForm struct {
// CtxPool is the engine context pool the form starts with, and the one
// the memory figure assumes.
CtxPool int
// Context is the context length the form starts with.
Context int
// StrataFamily and StrataSize are Strata's --family and --size, for a
// version of a family Strata runs, served on it.
StrataFamily, StrataSize string
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
@@ -122,6 +131,35 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
st.BytesTotal = total
}
f.Context = 4096
// Files of a family Strata runs, on a fleet that has Strata: served on
// it, from the version's directory, whether or not llama.cpp knows them.
if f.Engine == "llamacpp" && strataInstalled(fleet) {
if id, _, _, ok := hf.FileSource(st.Source, hf.DefaultEndpoint); ok {
if fam, ok := strata.ForRepo(id); ok {
if size, ok := strata.SizeForFile(fam.Name, st.File); ok {
f.Engine, f.StrataFamily, f.StrataSize = "strata", fam.Name, size.Name
f.Source = strings.TrimSuffix(mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/"
f.Context, f.Profile = 32768, "gpu"
f.Placements, f.FreeGPUs = placementsFor(l, fleet, f.Engine)
for _, pc := range f.Placements {
if pc.Selected {
f.Profile = pc.profile()
}
}
f.GPUs = 1
f.MinMemoryGB = size.MemoryGB()
f.MemoryBasis = l.T("serve.basis.strata", fam.Title, size.Name)
if cat != nil && cat.Declared(st.Model) {
f.Already = true
f.Refusal = l.T("serve.refuse.declared", st.Model)
}
return f
}
}
}
}
switch f.Engine {
case "":
f.Refusal = l.T("serve.refuse.no_engine", extOf(st.File))
@@ -220,6 +258,19 @@ func extOf(file string) string {
return file
}
// strataInstalled reports whether a node of the fleet has Strata.
func strataInstalled(fleet Source) bool {
if fleet == nil {
return false
}
for _, n := range fleet.Nodes() {
if slices.Contains(n.Engines, "strata") {
return true
}
}
return false
}
// runsAnywhere reports whether some node has this engine.
//
// True when nothing is known, so a fleet of agents older than the engine probe
@@ -304,6 +355,14 @@ func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
}
place := placeChoice{Value: strings.TrimSpace(get("placement"))}
switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); {
case onGPU && class != "" && engine == "strata":
// One card; the model's family and size, and the context.
p.GPUClass, p.GPUsPerNode, place.GPUs = class, 1, 1
family, size := strings.TrimSpace(get("strata_family")), strings.TrimSpace(get("strata_size"))
if _, err := strata.SizeOf(family, size); err != nil {
return m, err
}
p.Args = []string{"--family", family, "--size", size, "--context", strconv.Itoa(ctxLen), "--vision", "no"}
case onGPU && class != "":
// Placement is decided by gpu_class, which is what the selector
// sets; the count is gpus_per_node for llama.cpp, which spreads
@@ -330,7 +389,7 @@ func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
// The pool, not the per-request limit: llama.cpp divides it across its
// parallel slots, so the number a client may use is smaller and is what
// context_length above says.
if v := strings.TrimSpace(get("ctx_size")); v != "" {
if v := strings.TrimSpace(get("ctx_size")); v != "" && engine != "strata" {
if _, err := positive(l, v, "serve.what.ctx_pool"); err != nil {
return m, err
}
+36
View File
@@ -12,6 +12,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
@@ -554,3 +555,38 @@ func (f *fakeCatalog) Stop(_ context.Context, name, _ string) (catalog.Model, er
func (f *fakeCatalog) Start(_ context.Context, name, _ string) (catalog.Model, error) {
return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err
}
// A version of a family Strata runs, on a fleet with Strata, is served on it:
// one card, its family and size, the experts' memory, from the directory.
func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) {
st := held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf")
st.Source = "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF/resolve/abc/IQ2_XS/" + st.File
files := []mirror.Status{st, held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf")}
f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp", "strata"}, memoryGB: 251},
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} })
if f.Engine != "strata" || f.StrataFamily != "qwen" || f.StrataSize != "IQ2_XS" || f.Refusal != "" ||
f.MinMemoryGB != 52 || f.Context != 32768 || !strings.HasSuffix(f.Source, "/q38/abc123/") {
t.Fatalf("%+v", f)
}
// Without Strata on any node, llama.cpp as before, and its refusal.
if f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp"}},
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} }); f.Engine != "llamacpp" || f.Refusal == "" {
t.Errorf("%+v", f)
}
form := url.Values{"name": {"q38"}, "source": {f.Source}, "engine": {"strata"}, "replicas": {"1"},
"context_length": {"32768"}, "placement": {"gpu:6000-24g"}, "min_memory_gb": {"52"},
"strata_family": {"qwen"}, "strata_size": {"IQ2_XS"}}
m, err := buildModel(english, form.Get)
if err != nil {
t.Fatal(err)
}
p := m.Profiles[0]
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
strings.Join(p.Args, " ") != "--family qwen --size IQ2_XS --context 32768 --vision no" {
t.Errorf("%+v", p)
}
form.Set("strata_size", "IQ9")
if _, err := buildModel(english, form.Get); err == nil {
t.Error("an unknown size was taken")
}
}
+39
View File
@@ -9,6 +9,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
)
// The explorer lists what this cluster can run, judged by the nodes it has
@@ -137,8 +138,43 @@ var pictureTasks = []string{"text-to-image", "image-to-image", "image-to-3d", "t
// people want. A repository of Q8 only is judged kindly; its page says.
const ggufBitsPerWeight = 4.8
// strataLayout is how a file of a family Strata runs would run on it here,
// and whether it could.
func (c *clusterFit) strataLayout(l *Lang, repo, name string, bytes int64) (string, bool) {
if c == nil {
return "", false
}
fam, ok := strata.ForRepo(repo)
if !ok {
return "", false
}
opts := modelplan.StrataOptions(fam.Name, []modelplan.Artifact{{Name: name, Format: "gguf", Bytes: bytes}}, c.pool)
if len(opts) == 0 {
return "", false
}
return l.T("explore.strata", opts[0].Class, l.Num(opts[0].NeedGB, 0)), true
}
// strataRuns is the smallest of a Strata family's sizes that runs here, as
// a layout, for a repository of the family.
func (c *clusterFit) strataRuns(l *Lang, repo string) (string, bool) {
fam, ok := strata.ForRepo(repo)
if !ok {
return "", false
}
for _, s := range strata.SizesOf(fam.Name) {
if layout, ok := c.strataLayout(l, repo, s.Name+".gguf", int64(s.DownloadGB*1e9)); ok {
return layout, true
}
}
return "", false
}
// runs reports whether some node here could hold the repository's model.
func (c *clusterFit) runs(m hf.Model) bool {
if _, ok := c.strataRuns(english, m.ID); ok {
return true
}
params := m.Params()
if slices.Contains(pictureTasks, m.PipelineTag) || m.HasTag("diffusers") {
b := m.WeightBytes()
@@ -329,6 +365,9 @@ func layoutLabel(l *Lang, o modelplan.Option, links modelplan.Links) string {
// rowGPU is a listed repository's speed on the cards: its GGUF at Q4_K_M
// on llama.cpp, its safetensors on vLLM, whichever is faster here.
func (c *clusterFit) rowGPU(l *Lang, m hf.Model) (string, string) {
if layout, ok := c.strataRuns(l, m.ID); ok {
return "", layout
}
if !c.gpus() || m.Params() <= 0 {
return "", ""
}
+2 -1
View File
@@ -256,5 +256,6 @@
"mirror.err.not_adapter": "that file is a model, not a LoRA adapter",
"mirror.err.wrong_arch": "that adapter is for %s models and %s is %s",
"mirror.msg.adapter_added": "%[1]s: adapter %[2]s added; ask for it by that name once the replicas are restarted (llmgw ctl restart --model %[1]s)",
"mirror.err.adapter_op": "add or remove an adapter"
"mirror.err.adapter_op": "add or remove an adapter",
"explore.strata": "1× %s with Strata, %s GB of RAM"
}
+3 -1
View File
@@ -81,5 +81,7 @@
"serve.replicas_n.one": "%d replica",
"serve.replicas_n.other": "%d replicas",
"serve.msg.judge": "It is the guardrails' judge (role: guard): policies naming %s ask it; clients do not see it.",
"serve.msg.judge_renamed": "Its weights are a judge's, so it is the guardrails' judge (role: guard), named %[2]s as the policies ask rather than %[1]s."
"serve.msg.judge_renamed": "Its weights are a judge's, so it is the guardrails' judge (role: guard), named %[2]s as the policies ask rather than %[1]s.",
"serve.basis.strata": "what Strata keeps of %s %s's experts in RAM, and 4 GB for its engine and server",
"serve.strata": "Strata runs this (%s, %s): one card, and the experts in the node's RAM. The first start prepares the model on the node, which takes minutes."
}
+2 -1
View File
@@ -249,5 +249,6 @@
"mirror.err.not_adapter": "bu dosya bir LoRA adaptörü değil, bir model",
"mirror.err.wrong_arch": "bu adaptör %s modelleri için; %s ise %s",
"mirror.msg.adapter_added": "%[1]s: %[2]s adaptörü eklendi; replikalar yeniden başlatılınca bu adla isteyin (llmgw ctl restart --model %[1]s)",
"mirror.err.adapter_op": "bir adaptör ekleyin ya da kaldırın"
"mirror.err.adapter_op": "bir adaptör ekleyin ya da kaldırın",
"explore.strata": "Strata ile 1× %s, %s GB RAM"
}
+3 -1
View File
@@ -78,5 +78,7 @@
"serve.msg.added": "%s kataloğa eklendi, %s: %s; bir sonraki geçişte yerleştirilecek",
"serve.replicas_n.other": "%d replika",
"serve.msg.judge": "Guardrail'lerin judge modeli olarak eklendi (role: guard): %s adını veren politikalar ona soruyor; istemciler görmüyor.",
"serve.msg.judge_renamed": "Ağırlıkları bir judge modelinin olduğu için guardrail judge'ı olarak eklendi (role: guard); adı %[1]s yerine politikaların istediği gibi %[2]s."
"serve.msg.judge_renamed": "Ağırlıkları bir judge modelinin olduğu için guardrail judge'ı olarak eklendi (role: guard); adı %[1]s yerine politikaların istediği gibi %[2]s.",
"serve.basis.strata": "Strata'nın %s %s için RAM'de tuttuğu uzmanlar, artı motoru ve sunucusu için 4 GB",
"serve.strata": "Bunu Strata çalıştırır (%s, %s): tek kart, uzmanlar node'un RAM'inde. İlk açılışta model node üzerinde hazırlanır; bu birkaç dakika sürer."
}
+8 -5
View File
@@ -133,8 +133,11 @@
<input type="hidden" name="csrf" value="{{$.CSRF}}">
<input type="hidden" name="source" value="{{.Source}}">
<input type="hidden" name="engine" value="{{.Engine}}">
{{if .StrataFamily}}<input type="hidden" name="strata_family" value="{{.StrataFamily}}">
<input type="hidden" name="strata_size" value="{{.StrataSize}}">{{end}}
<p class="sub">{{th "serve.serving" .Source .Engine}}
{{if .Files}}{{t "serve.whole_version" .Files}}{{end}}</p>
{{if .StrataFamily}}<p class="note">{{t "serve.strata" .StrataFamily .StrataSize}}</p>{{end}}
<div class="field-row">
<label>{{t "serve.name"}} <input type="text" name="name" value="{{.Model}}" required></label>
<label>{{t "serve.aliases"}} <span class="opt">{{t "serve.aliases_hint"}}</span>
@@ -154,11 +157,11 @@
placeholder="{{t "serve.description_example"}}"></label>
<div class="field-row">
<label>{{t "serve.context_length"}}
<input type="text" name="context_length" value="4096" required></label>
<input type="text" name="context_length" value="{{.Context}}" required></label>
<label>{{t "serve.max_output"}} <span class="opt">{{t "serve.optional"}}</span>
<input type="text" name="max_output" value="1024"></label>
<label>{{t "serve.ctx_pool"}} <span class="opt">{{t "serve.ctx_pool_hint"}}</span>
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
{{if ne .Engine "strata"}}<label>{{t "serve.ctx_pool"}} <span class="opt">{{t "serve.ctx_pool_hint"}}</span>
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>{{end}}
</div>
{{/* Where it runs is a choice with consequences, so it is a choice: the
GPU classes the fleet has, and the CPU for llama.cpp. A profile name
@@ -169,8 +172,8 @@
<select name="placement">
{{range .Placements}}<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>{{end}}
</select></label>
<label>{{t "serve.gpus"}} <span class="opt">{{t "serve.gpus_hint"}}</span>
<input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>
{{if ne .Engine "strata"}}<label>{{t "serve.gpus"}} <span class="opt">{{t "serve.gpus_hint"}}</span>
<input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>{{end}}
{{if eq .Engine "llamacpp"}}<label>{{t "serve.parallel"}} <span class="opt">{{t "serve.parallel_hint"}}</span>
<input type="text" name="parallel" value="{{.Parallel}}" size="3"></label>{{end}}
</div>