The mirror page's Strata tab; Strata's ready-made packs served with Strata fetching them, strata-ple named
CI / test (push) Successful in 10m35s
CI / test (push) Successful in 10m35s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4ee9ba54ef
commit
845de8eef4
13 files changed
+513
-21
No files matched your search
@@ -0,0 +1,153 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
||||
)
|
||||
|
||||
// The mirror page's Strata tab: the models Strata runs, each size with what
|
||||
// it downloads and keeps in RAM and where it would run here, and a button
|
||||
// that fetches a size's files into the mirror, where the version's serve
|
||||
// form then serves it on Strata.
|
||||
|
||||
type strataView struct {
|
||||
// Nodes are the nodes that have Strata installed.
|
||||
Nodes []string
|
||||
Families []strataFamilyRow
|
||||
}
|
||||
|
||||
type strataFamilyRow struct {
|
||||
Name, Title, Repo string
|
||||
// Fetchable is a family whose files llmgw fetches into the mirror;
|
||||
// FetchesItself one whose files Strata fetches onto the node, served
|
||||
// from here at once.
|
||||
Fetchable, FetchesItself bool
|
||||
Sizes []strataSizeRow
|
||||
}
|
||||
|
||||
type strataSizeRow struct {
|
||||
Size string
|
||||
DownloadGB, RAMGB string
|
||||
Runs string
|
||||
Fits, NVIDIAOnly bool
|
||||
}
|
||||
|
||||
func (p *AccountPages) strataView(l *Lang) *strataView {
|
||||
v := &strataView{}
|
||||
var pool []modelplan.PoolNode
|
||||
if fit := p.fitOf(); fit != nil {
|
||||
pool = fit.pool
|
||||
}
|
||||
for _, n := range pool {
|
||||
if slices.Contains(n.Engines, "strata") {
|
||||
v.Nodes = append(v.Nodes, n.ID)
|
||||
}
|
||||
}
|
||||
for _, f := range strata.Families {
|
||||
row := strataFamilyRow{Name: f.Name, Title: f.Title, Repo: f.Repo, Fetchable: f.Fetchable(),
|
||||
FetchesItself: f.FetchesItself()}
|
||||
for _, s := range strata.SizesOf(f.Name) {
|
||||
sr := strataSizeRow{Size: s.Name, DownloadGB: l.Num(s.DownloadGB, 0), RAMGB: l.Int(int64(s.MemoryGB())),
|
||||
NVIDIAOnly: s.NVIDIAOnly}
|
||||
opts := modelplan.StrataOptions(f.Name, []modelplan.Artifact{{Name: s.Name + ".gguf", Format: "gguf",
|
||||
Bytes: int64(s.DownloadGB * 1e9)}}, pool)
|
||||
switch {
|
||||
case len(opts) > 0:
|
||||
sr.Fits, sr.Runs = true, l.T("explore.strata", opts[0].Class, l.Num(opts[0].NeedGB, 0))
|
||||
case len(v.Nodes) == 0:
|
||||
sr.Runs = l.T("strata.no_node")
|
||||
default:
|
||||
sr.Runs = l.T("strata.no_room", s.MemoryGB())
|
||||
}
|
||||
row.Sizes = append(row.Sizes, sr)
|
||||
}
|
||||
v.Families = append(v.Families, row)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// serveStrata adds a model of a family Strata fetches itself: its source
|
||||
// the repository, its replica on a card of the class the plan found.
|
||||
func (p *AccountPages) serveStrata(ctx context.Context, l *Lang, by, family, size string) (string, error) {
|
||||
fam, ok := strata.FamilyNamed(family)
|
||||
s, err := strata.SizeOf(family, size)
|
||||
if !ok || err != nil {
|
||||
return "", errors.New(l.T("strata.err.unknown", family, size))
|
||||
}
|
||||
if !fam.FetchesItself() || p.catalog == nil {
|
||||
return "", errors.New(l.T("strata.err.not_servable", fam.Title))
|
||||
}
|
||||
var pool []modelplan.PoolNode
|
||||
if fit := p.fitOf(); fit != nil {
|
||||
pool = fit.pool
|
||||
}
|
||||
opts := modelplan.StrataOptions(fam.Name, []modelplan.Artifact{{Name: s.Name + ".gguf", Format: "gguf"}}, pool)
|
||||
if len(opts) == 0 {
|
||||
return "", errors.New(l.T("strata.no_room", s.MemoryGB()))
|
||||
}
|
||||
name := fam.Model + "-" + strings.ToLower(s.Name)
|
||||
disk := s.DownloadGB
|
||||
if s.DiskGB > disk {
|
||||
disk = s.DiskGB
|
||||
}
|
||||
m := catalog.Model{Name: name, Source: strata.RepoOf(fam, s), RoutingProfile: "chat", ContextLength: strataContext,
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "strata-" + opts[0].Class, Engine: "strata", GPUClass: opts[0].Class,
|
||||
GPUsPerNode: 1, Weight: 1, Priority: 100, MinMemoryGB: s.MemoryGB(), MinDiskGB: int(disk) + 10,
|
||||
Args: []string{"--family", fam.Name, "--size", s.Name, "--context", strconv.Itoa(strataContext), "--vision", "no"},
|
||||
// Its files come onto the node on its first start, tens of
|
||||
// gigabytes from the internet, before it is prepared.
|
||||
StartupTimeout: 4 * time.Hour}}}
|
||||
if err := p.catalog.Add(ctx, m, by); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return l.T("strata.msg.served", name, l.Num(disk, 0)), nil
|
||||
}
|
||||
|
||||
// strataContext is the context a Strata model is served with from here.
|
||||
const strataContext = 32768
|
||||
|
||||
// fetchStrata fetches a Strata size's files from its family's repository
|
||||
// into the mirror, each checked against the sha256 the registry published.
|
||||
func (p *AccountPages) fetchStrata(ctx context.Context, l *Lang, m MirrorAdmin, family, size string) (string, error) {
|
||||
fam, ok := strata.FamilyNamed(family)
|
||||
if _, err := strata.SizeOf(family, size); !ok || err != nil {
|
||||
return "", errors.New(l.T("strata.err.unknown", family, size))
|
||||
}
|
||||
if !fam.Fetchable() || p.registry == nil {
|
||||
return "", errors.New(l.T("strata.err.not_fetchable", fam.Title))
|
||||
}
|
||||
repo, err := p.registry.Repo(ctx, fam.Repo)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var paths []string
|
||||
sums := map[string]string{}
|
||||
for _, f := range repo.Files {
|
||||
paths = append(paths, f.Path)
|
||||
sums[f.Path] = f.SHA256
|
||||
}
|
||||
files := strata.FilesOf(size, paths)
|
||||
if len(files) == 0 {
|
||||
return "", errors.New(l.T("strata.err.no_files", fam.Repo, size))
|
||||
}
|
||||
model := fam.Model + "-" + strings.ToLower(size)
|
||||
version := repo.ShortRevision()
|
||||
for i, path := range files {
|
||||
req := mirror.FetchRequest{Source: repo.DownloadURL("", path), Model: model, Version: version,
|
||||
File: mirror.LastPathElement(path), SHA256: sums[path]}
|
||||
if _, err := m.Fetch(ctx, req); err != nil && !strings.Contains(err.Error(), "already") {
|
||||
return "", errors.New(l.T("mirror.err.partly_started", i, len(files), req.File, l.Err(err)))
|
||||
}
|
||||
}
|
||||
return l.T("mirror.msg.started", model, version, l.N("mirror.files", int64(len(files)))), nil
|
||||
}
|
||||
Reference in new issue
Block a user