1 parent
7465c24830
commit
d6f8f433f4
24 files changed
+437
-22
No files matched your search
@@ -4,14 +4,18 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
||||
)
|
||||
|
||||
// CatalogAdmin adds a model to the catalogue in force.
|
||||
@@ -76,6 +80,11 @@ type serveForm struct {
|
||||
// CtxPool is the engine context pool the form starts with, and the one
|
||||
// the memory figure assumes.
|
||||
CtxPool int
|
||||
// Context is the context length the form starts with.
|
||||
Context int
|
||||
// StrataFamily and StrataSize are Strata's --family and --size, for a
|
||||
// version of a family Strata runs, served on it.
|
||||
StrataFamily, StrataSize string
|
||||
|
||||
// Refusal is set when this version cannot be served as it stands, and says
|
||||
// why. Offered and then failing at placement is worse than not offered.
|
||||
@@ -122,6 +131,35 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
|
||||
st.BytesTotal = total
|
||||
}
|
||||
|
||||
f.Context = 4096
|
||||
// Files of a family Strata runs, on a fleet that has Strata: served on
|
||||
// it, from the version's directory, whether or not llama.cpp knows them.
|
||||
if f.Engine == "llamacpp" && strataInstalled(fleet) {
|
||||
if id, _, _, ok := hf.FileSource(st.Source, hf.DefaultEndpoint); ok {
|
||||
if fam, ok := strata.ForRepo(id); ok {
|
||||
if size, ok := strata.SizeForFile(fam.Name, st.File); ok {
|
||||
f.Engine, f.StrataFamily, f.StrataSize = "strata", fam.Name, size.Name
|
||||
f.Source = strings.TrimSuffix(mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/"
|
||||
f.Context, f.Profile = 32768, "gpu"
|
||||
f.Placements, f.FreeGPUs = placementsFor(l, fleet, f.Engine)
|
||||
for _, pc := range f.Placements {
|
||||
if pc.Selected {
|
||||
f.Profile = pc.profile()
|
||||
}
|
||||
}
|
||||
f.GPUs = 1
|
||||
f.MinMemoryGB = size.MemoryGB()
|
||||
f.MemoryBasis = l.T("serve.basis.strata", fam.Title, size.Name)
|
||||
if cat != nil && cat.Declared(st.Model) {
|
||||
f.Already = true
|
||||
f.Refusal = l.T("serve.refuse.declared", st.Model)
|
||||
}
|
||||
return f
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch f.Engine {
|
||||
case "":
|
||||
f.Refusal = l.T("serve.refuse.no_engine", extOf(st.File))
|
||||
@@ -220,6 +258,19 @@ func extOf(file string) string {
|
||||
return file
|
||||
}
|
||||
|
||||
// strataInstalled reports whether a node of the fleet has Strata.
|
||||
func strataInstalled(fleet Source) bool {
|
||||
if fleet == nil {
|
||||
return false
|
||||
}
|
||||
for _, n := range fleet.Nodes() {
|
||||
if slices.Contains(n.Engines, "strata") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// runsAnywhere reports whether some node has this engine.
|
||||
//
|
||||
// True when nothing is known, so a fleet of agents older than the engine probe
|
||||
@@ -304,6 +355,14 @@ func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
|
||||
}
|
||||
place := placeChoice{Value: strings.TrimSpace(get("placement"))}
|
||||
switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); {
|
||||
case onGPU && class != "" && engine == "strata":
|
||||
// One card; the model's family and size, and the context.
|
||||
p.GPUClass, p.GPUsPerNode, place.GPUs = class, 1, 1
|
||||
family, size := strings.TrimSpace(get("strata_family")), strings.TrimSpace(get("strata_size"))
|
||||
if _, err := strata.SizeOf(family, size); err != nil {
|
||||
return m, err
|
||||
}
|
||||
p.Args = []string{"--family", family, "--size", size, "--context", strconv.Itoa(ctxLen), "--vision", "no"}
|
||||
case onGPU && class != "":
|
||||
// Placement is decided by gpu_class, which is what the selector
|
||||
// sets; the count is gpus_per_node for llama.cpp, which spreads
|
||||
@@ -330,7 +389,7 @@ func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
|
||||
// The pool, not the per-request limit: llama.cpp divides it across its
|
||||
// parallel slots, so the number a client may use is smaller and is what
|
||||
// context_length above says.
|
||||
if v := strings.TrimSpace(get("ctx_size")); v != "" {
|
||||
if v := strings.TrimSpace(get("ctx_size")); v != "" && engine != "strata" {
|
||||
if _, err := positive(l, v, "serve.what.ctx_pool"); err != nil {
|
||||
return m, err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user