1 parent
7465c24830
commit
d6f8f433f4
24 files changed
+437
-22
No files matched your search
@@ -823,7 +823,11 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
Version: got.ShortRevision(),
|
||||
}
|
||||
rs.Verdict = p.verdictFor(langOf(r), got.Architecture, set.Name)
|
||||
if fit != nil && rs.Verdict.Servable {
|
||||
// A family Strata runs: one card and the experts in RAM, whether
|
||||
// or not llama.cpp could load it.
|
||||
if layout, ok := fit.strataLayout(langOf(r), got.ID, set.Name, set.Bytes); ok {
|
||||
rs.Verdict, rs.GPULayout = gguf.Verdict{Checked: true, Servable: true}, layout
|
||||
} else if fit != nil && rs.Verdict.Servable {
|
||||
if v, ok := fit.holds(langOf(r), set.Name, set.Bytes, got.Params); !ok {
|
||||
rs.Verdict = v
|
||||
} else if gguf.IsGGUF(set.Name) {
|
||||
|
||||
@@ -4,14 +4,18 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
||||
)
|
||||
|
||||
// CatalogAdmin adds a model to the catalogue in force.
|
||||
@@ -76,6 +80,11 @@ type serveForm struct {
|
||||
// CtxPool is the engine context pool the form starts with, and the one
|
||||
// the memory figure assumes.
|
||||
CtxPool int
|
||||
// Context is the context length the form starts with.
|
||||
Context int
|
||||
// StrataFamily and StrataSize are Strata's --family and --size, for a
|
||||
// version of a family Strata runs, served on it.
|
||||
StrataFamily, StrataSize string
|
||||
|
||||
// Refusal is set when this version cannot be served as it stands, and says
|
||||
// why. Offered and then failing at placement is worse than not offered.
|
||||
@@ -122,6 +131,35 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
|
||||
st.BytesTotal = total
|
||||
}
|
||||
|
||||
f.Context = 4096
|
||||
// Files of a family Strata runs, on a fleet that has Strata: served on
|
||||
// it, from the version's directory, whether or not llama.cpp knows them.
|
||||
if f.Engine == "llamacpp" && strataInstalled(fleet) {
|
||||
if id, _, _, ok := hf.FileSource(st.Source, hf.DefaultEndpoint); ok {
|
||||
if fam, ok := strata.ForRepo(id); ok {
|
||||
if size, ok := strata.SizeForFile(fam.Name, st.File); ok {
|
||||
f.Engine, f.StrataFamily, f.StrataSize = "strata", fam.Name, size.Name
|
||||
f.Source = strings.TrimSuffix(mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/"
|
||||
f.Context, f.Profile = 32768, "gpu"
|
||||
f.Placements, f.FreeGPUs = placementsFor(l, fleet, f.Engine)
|
||||
for _, pc := range f.Placements {
|
||||
if pc.Selected {
|
||||
f.Profile = pc.profile()
|
||||
}
|
||||
}
|
||||
f.GPUs = 1
|
||||
f.MinMemoryGB = size.MemoryGB()
|
||||
f.MemoryBasis = l.T("serve.basis.strata", fam.Title, size.Name)
|
||||
if cat != nil && cat.Declared(st.Model) {
|
||||
f.Already = true
|
||||
f.Refusal = l.T("serve.refuse.declared", st.Model)
|
||||
}
|
||||
return f
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch f.Engine {
|
||||
case "":
|
||||
f.Refusal = l.T("serve.refuse.no_engine", extOf(st.File))
|
||||
@@ -220,6 +258,19 @@ func extOf(file string) string {
|
||||
return file
|
||||
}
|
||||
|
||||
// strataInstalled reports whether a node of the fleet has Strata.
|
||||
func strataInstalled(fleet Source) bool {
|
||||
if fleet == nil {
|
||||
return false
|
||||
}
|
||||
for _, n := range fleet.Nodes() {
|
||||
if slices.Contains(n.Engines, "strata") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// runsAnywhere reports whether some node has this engine.
|
||||
//
|
||||
// True when nothing is known, so a fleet of agents older than the engine probe
|
||||
@@ -304,6 +355,14 @@ func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
|
||||
}
|
||||
place := placeChoice{Value: strings.TrimSpace(get("placement"))}
|
||||
switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); {
|
||||
case onGPU && class != "" && engine == "strata":
|
||||
// One card; the model's family and size, and the context.
|
||||
p.GPUClass, p.GPUsPerNode, place.GPUs = class, 1, 1
|
||||
family, size := strings.TrimSpace(get("strata_family")), strings.TrimSpace(get("strata_size"))
|
||||
if _, err := strata.SizeOf(family, size); err != nil {
|
||||
return m, err
|
||||
}
|
||||
p.Args = []string{"--family", family, "--size", size, "--context", strconv.Itoa(ctxLen), "--vision", "no"}
|
||||
case onGPU && class != "":
|
||||
// Placement is decided by gpu_class, which is what the selector
|
||||
// sets; the count is gpus_per_node for llama.cpp, which spreads
|
||||
@@ -330,7 +389,7 @@ func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
|
||||
// The pool, not the per-request limit: llama.cpp divides it across its
|
||||
// parallel slots, so the number a client may use is smaller and is what
|
||||
// context_length above says.
|
||||
if v := strings.TrimSpace(get("ctx_size")); v != "" {
|
||||
if v := strings.TrimSpace(get("ctx_size")); v != "" && engine != "strata" {
|
||||
if _, err := positive(l, v, "serve.what.ctx_pool"); err != nil {
|
||||
return m, err
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
@@ -554,3 +555,38 @@ func (f *fakeCatalog) Stop(_ context.Context, name, _ string) (catalog.Model, er
|
||||
func (f *fakeCatalog) Start(_ context.Context, name, _ string) (catalog.Model, error) {
|
||||
return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err
|
||||
}
|
||||
|
||||
// A version of a family Strata runs, on a fleet with Strata, is served on it:
|
||||
// one card, its family and size, the experts' memory, from the directory.
|
||||
func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) {
|
||||
st := held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf")
|
||||
st.Source = "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF/resolve/abc/IQ2_XS/" + st.File
|
||||
files := []mirror.Status{st, held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf")}
|
||||
f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp", "strata"}, memoryGB: 251},
|
||||
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} })
|
||||
if f.Engine != "strata" || f.StrataFamily != "qwen" || f.StrataSize != "IQ2_XS" || f.Refusal != "" ||
|
||||
f.MinMemoryGB != 52 || f.Context != 32768 || !strings.HasSuffix(f.Source, "/q38/abc123/") {
|
||||
t.Fatalf("%+v", f)
|
||||
}
|
||||
// Without Strata on any node, llama.cpp as before, and its refusal.
|
||||
if f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp"}},
|
||||
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} }); f.Engine != "llamacpp" || f.Refusal == "" {
|
||||
t.Errorf("%+v", f)
|
||||
}
|
||||
form := url.Values{"name": {"q38"}, "source": {f.Source}, "engine": {"strata"}, "replicas": {"1"},
|
||||
"context_length": {"32768"}, "placement": {"gpu:6000-24g"}, "min_memory_gb": {"52"},
|
||||
"strata_family": {"qwen"}, "strata_size": {"IQ2_XS"}}
|
||||
m, err := buildModel(english, form.Get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := m.Profiles[0]
|
||||
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
|
||||
strings.Join(p.Args, " ") != "--family qwen --size IQ2_XS --context 32768 --vision no" {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
form.Set("strata_size", "IQ9")
|
||||
if _, err := buildModel(english, form.Get); err == nil {
|
||||
t.Error("an unknown size was taken")
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
||||
)
|
||||
|
||||
// The explorer lists what this cluster can run, judged by the nodes it has
|
||||
@@ -137,8 +138,43 @@ var pictureTasks = []string{"text-to-image", "image-to-image", "image-to-3d", "t
|
||||
// people want. A repository of Q8 only is judged kindly; its page says.
|
||||
const ggufBitsPerWeight = 4.8
|
||||
|
||||
// strataLayout is how a file of a family Strata runs would run on it here,
|
||||
// and whether it could.
|
||||
func (c *clusterFit) strataLayout(l *Lang, repo, name string, bytes int64) (string, bool) {
|
||||
if c == nil {
|
||||
return "", false
|
||||
}
|
||||
fam, ok := strata.ForRepo(repo)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
opts := modelplan.StrataOptions(fam.Name, []modelplan.Artifact{{Name: name, Format: "gguf", Bytes: bytes}}, c.pool)
|
||||
if len(opts) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return l.T("explore.strata", opts[0].Class, l.Num(opts[0].NeedGB, 0)), true
|
||||
}
|
||||
|
||||
// strataRuns is the smallest of a Strata family's sizes that runs here, as
|
||||
// a layout, for a repository of the family.
|
||||
func (c *clusterFit) strataRuns(l *Lang, repo string) (string, bool) {
|
||||
fam, ok := strata.ForRepo(repo)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
for _, s := range strata.SizesOf(fam.Name) {
|
||||
if layout, ok := c.strataLayout(l, repo, s.Name+".gguf", int64(s.DownloadGB*1e9)); ok {
|
||||
return layout, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// runs reports whether some node here could hold the repository's model.
|
||||
func (c *clusterFit) runs(m hf.Model) bool {
|
||||
if _, ok := c.strataRuns(english, m.ID); ok {
|
||||
return true
|
||||
}
|
||||
params := m.Params()
|
||||
if slices.Contains(pictureTasks, m.PipelineTag) || m.HasTag("diffusers") {
|
||||
b := m.WeightBytes()
|
||||
@@ -329,6 +365,9 @@ func layoutLabel(l *Lang, o modelplan.Option, links modelplan.Links) string {
|
||||
// rowGPU is a listed repository's speed on the cards: its GGUF at Q4_K_M
|
||||
// on llama.cpp, its safetensors on vLLM, whichever is faster here.
|
||||
func (c *clusterFit) rowGPU(l *Lang, m hf.Model) (string, string) {
|
||||
if layout, ok := c.strataRuns(l, m.ID); ok {
|
||||
return "", layout
|
||||
}
|
||||
if !c.gpus() || m.Params() <= 0 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
@@ -256,5 +256,6 @@
|
||||
"mirror.err.not_adapter": "that file is a model, not a LoRA adapter",
|
||||
"mirror.err.wrong_arch": "that adapter is for %s models and %s is %s",
|
||||
"mirror.msg.adapter_added": "%[1]s: adapter %[2]s added; ask for it by that name once the replicas are restarted (llmgw ctl restart --model %[1]s)",
|
||||
"mirror.err.adapter_op": "add or remove an adapter"
|
||||
"mirror.err.adapter_op": "add or remove an adapter",
|
||||
"explore.strata": "1× %s with Strata, %s GB of RAM"
|
||||
}
|
||||
@@ -81,5 +81,7 @@
|
||||
"serve.replicas_n.one": "%d replica",
|
||||
"serve.replicas_n.other": "%d replicas",
|
||||
"serve.msg.judge": "It is the guardrails' judge (role: guard): policies naming %s ask it; clients do not see it.",
|
||||
"serve.msg.judge_renamed": "Its weights are a judge's, so it is the guardrails' judge (role: guard), named %[2]s as the policies ask rather than %[1]s."
|
||||
"serve.msg.judge_renamed": "Its weights are a judge's, so it is the guardrails' judge (role: guard), named %[2]s as the policies ask rather than %[1]s.",
|
||||
"serve.basis.strata": "what Strata keeps of %s %s's experts in RAM, and 4 GB for its engine and server",
|
||||
"serve.strata": "Strata runs this (%s, %s): one card, and the experts in the node's RAM. The first start prepares the model on the node, which takes minutes."
|
||||
}
|
||||
@@ -249,5 +249,6 @@
|
||||
"mirror.err.not_adapter": "bu dosya bir LoRA adaptörü değil, bir model",
|
||||
"mirror.err.wrong_arch": "bu adaptör %s modelleri için; %s ise %s",
|
||||
"mirror.msg.adapter_added": "%[1]s: %[2]s adaptörü eklendi; replikalar yeniden başlatılınca bu adla isteyin (llmgw ctl restart --model %[1]s)",
|
||||
"mirror.err.adapter_op": "bir adaptör ekleyin ya da kaldırın"
|
||||
"mirror.err.adapter_op": "bir adaptör ekleyin ya da kaldırın",
|
||||
"explore.strata": "Strata ile 1× %s, %s GB RAM"
|
||||
}
|
||||
@@ -78,5 +78,7 @@
|
||||
"serve.msg.added": "%s kataloğa eklendi, %s: %s; bir sonraki geçişte yerleştirilecek",
|
||||
"serve.replicas_n.other": "%d replika",
|
||||
"serve.msg.judge": "Guardrail'lerin judge modeli olarak eklendi (role: guard): %s adını veren politikalar ona soruyor; istemciler görmüyor.",
|
||||
"serve.msg.judge_renamed": "Ağırlıkları bir judge modelinin olduğu için guardrail judge'ı olarak eklendi (role: guard); adı %[1]s yerine politikaların istediği gibi %[2]s."
|
||||
"serve.msg.judge_renamed": "Ağırlıkları bir judge modelinin olduğu için guardrail judge'ı olarak eklendi (role: guard); adı %[1]s yerine politikaların istediği gibi %[2]s.",
|
||||
"serve.basis.strata": "Strata'nın %s %s için RAM'de tuttuğu uzmanlar, artı motoru ve sunucusu için 4 GB",
|
||||
"serve.strata": "Bunu Strata çalıştırır (%s, %s): tek kart, uzmanlar node'un RAM'inde. İlk açılışta model node üzerinde hazırlanır; bu birkaç dakika sürer."
|
||||
}
|
||||
@@ -133,8 +133,11 @@
|
||||
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
||||
<input type="hidden" name="source" value="{{.Source}}">
|
||||
<input type="hidden" name="engine" value="{{.Engine}}">
|
||||
{{if .StrataFamily}}<input type="hidden" name="strata_family" value="{{.StrataFamily}}">
|
||||
<input type="hidden" name="strata_size" value="{{.StrataSize}}">{{end}}
|
||||
<p class="sub">{{th "serve.serving" .Source .Engine}}
|
||||
{{if .Files}}{{t "serve.whole_version" .Files}}{{end}}</p>
|
||||
{{if .StrataFamily}}<p class="note">{{t "serve.strata" .StrataFamily .StrataSize}}</p>{{end}}
|
||||
<div class="field-row">
|
||||
<label>{{t "serve.name"}} <input type="text" name="name" value="{{.Model}}" required></label>
|
||||
<label>{{t "serve.aliases"}} <span class="opt">{{t "serve.aliases_hint"}}</span>
|
||||
@@ -154,11 +157,11 @@
|
||||
placeholder="{{t "serve.description_example"}}"></label>
|
||||
<div class="field-row">
|
||||
<label>{{t "serve.context_length"}}
|
||||
<input type="text" name="context_length" value="4096" required></label>
|
||||
<input type="text" name="context_length" value="{{.Context}}" required></label>
|
||||
<label>{{t "serve.max_output"}} <span class="opt">{{t "serve.optional"}}</span>
|
||||
<input type="text" name="max_output" value="1024"></label>
|
||||
<label>{{t "serve.ctx_pool"}} <span class="opt">{{t "serve.ctx_pool_hint"}}</span>
|
||||
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
|
||||
{{if ne .Engine "strata"}}<label>{{t "serve.ctx_pool"}} <span class="opt">{{t "serve.ctx_pool_hint"}}</span>
|
||||
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>{{end}}
|
||||
</div>
|
||||
{{/* Where it runs is a choice with consequences, so it is a choice: the
|
||||
GPU classes the fleet has, and the CPU for llama.cpp. A profile name
|
||||
@@ -169,8 +172,8 @@
|
||||
<select name="placement">
|
||||
{{range .Placements}}<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>{{end}}
|
||||
</select></label>
|
||||
<label>{{t "serve.gpus"}} <span class="opt">{{t "serve.gpus_hint"}}</span>
|
||||
<input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>
|
||||
{{if ne .Engine "strata"}}<label>{{t "serve.gpus"}} <span class="opt">{{t "serve.gpus_hint"}}</span>
|
||||
<input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>{{end}}
|
||||
{{if eq .Engine "llamacpp"}}<label>{{t "serve.parallel"}} <span class="opt">{{t "serve.parallel_hint"}}</span>
|
||||
<input type="text" name="parallel" value="{{.Parallel}}" size="3"></label>{{end}}
|
||||
</div>
|
||||
|
||||
Reference in new issue
Block a user