CI / test (push) Failing after 6m37s
Each agent measures its node's memory read rate when it starts: every CPU sums a share of a buffer past any cache, a tenth of free memory at most, the best of four passes, in a fraction of a second; reported in Host (mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it uses once, so one reply's speed is about that rate over those bytes: modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token, calibrated on the PoC against llama.cpp's predicted_per_second (node01 24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s, qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active part (ActiveShare). The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that holds the model; a repository's page gives it for each GGUF by its size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
237 lines
7.4 KiB
Go
237 lines
7.4 KiB
Go
package web
|
|
|
|
import (
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
|
)
|
|
|
|
// The explorer lists what this cluster can run, judged by the nodes it has
|
|
// now rather than by a size picked from a list: a site with H100s sees the
|
|
// 70B models, one with a T4 and CPUs sees what fits there. From the
|
|
// registry's listing alone — a parameter count, safetensors by dtype, tags
|
|
// — so no repository is opened to decide; the repository's page sizes each
|
|
// file exactly.
|
|
|
|
// clusterFit is the hardware a listing is judged against.
|
|
type clusterFit struct {
|
|
pool []modelplan.PoolNode
|
|
// fp8 are the nodes with a card vLLM runs FP8 weights on, compute
|
|
// capability 8.9 or newer, as the agent requires.
|
|
fp8 []modelplan.PoolNode
|
|
// card is the largest card's memory in GB, cc its compute capability;
|
|
// host the most memory one node has.
|
|
card int
|
|
cc string
|
|
host int
|
|
// cpus are the nodes whose memory's speed their agent measured, for
|
|
// how fast a model would answer on one.
|
|
cpus []cpuHost
|
|
}
|
|
|
|
// cpuHost is a node as a model on its CPU sees it.
|
|
type cpuHost struct {
|
|
id string
|
|
ramGB int
|
|
bwGBs float64
|
|
}
|
|
|
|
// fp8Capability is the oldest card vLLM runs FP8 weights on.
|
|
const fp8Capability = 8.9
|
|
|
|
// fitOf is the cluster's hardware now, nil when there is none to judge by:
|
|
// no fleet to ask, or no node ready.
|
|
func (p *AccountPages) fitOf() *clusterFit {
|
|
if p.fleet == nil {
|
|
return nil
|
|
}
|
|
nodes := p.fleet.Nodes()
|
|
c := &clusterFit{pool: adminapi.PoolNodesOf(nodes)}
|
|
if len(c.pool) == 0 {
|
|
return nil
|
|
}
|
|
newest := map[string]float64{}
|
|
for _, n := range nodes {
|
|
if n.State == "ready" && n.MemBandwidthGBs > 0 {
|
|
c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
|
|
}
|
|
for _, s := range n.ComputeCapabilities {
|
|
if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] {
|
|
newest[n.ID] = v
|
|
}
|
|
}
|
|
}
|
|
for _, n := range c.pool {
|
|
c.host = max(c.host, n.HostMemGB)
|
|
if n.GPUs > 0 && n.GPUMemGB >= c.card {
|
|
if n.GPUMemGB > c.card || newest[n.ID] > atof(c.cc) {
|
|
c.cc = strconv.FormatFloat(newest[n.ID], 'f', 1, 64)
|
|
}
|
|
c.card = n.GPUMemGB
|
|
}
|
|
if newest[n.ID] >= fp8Capability {
|
|
c.fp8 = append(c.fp8, n)
|
|
}
|
|
}
|
|
return c
|
|
}
|
|
|
|
func atof(s string) float64 { v, _ := strconv.ParseFloat(s, 64); return v }
|
|
|
|
// gpus reports whether any node has a card.
|
|
func (c *clusterFit) gpus() bool { return c.card > 0 }
|
|
|
|
// pictureTasks are a pipeline ComfyUI runs, on a card, rather than an
|
|
// engine for text.
|
|
var pictureTasks = []string{"text-to-image", "image-to-image", "image-to-3d", "text-to-3d",
|
|
"text-to-video", "image-to-video", "unconditional-image-generation"}
|
|
|
|
// ggufBitsPerWeight is the quantisation a GGUF repository is judged by:
|
|
// Q4_K_M, the smallest of those published that answers as well as most
|
|
// people want. A repository of Q8 only is judged kindly; its page says.
|
|
const ggufBitsPerWeight = 4.8
|
|
|
|
// runs reports whether some node here could hold the repository's model.
|
|
func (c *clusterFit) runs(m hf.Model) bool {
|
|
params := m.Params()
|
|
if slices.Contains(pictureTasks, m.PipelineTag) || m.HasTag("diffusers") {
|
|
b := m.WeightBytes()
|
|
if b == 0 {
|
|
b = params * 2
|
|
}
|
|
// ComfyUI holds the model on one card, with room for its working.
|
|
return c.gpus() && float64(b+2<<30) <= float64(c.card)*0.9*(1<<30)
|
|
}
|
|
if m.GGUF != nil || m.HasTag("gguf") && m.Safetensors == nil {
|
|
if params == 0 {
|
|
return true
|
|
}
|
|
b := int64(float64(params) * ggufBitsPerWeight / 8)
|
|
if anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b,
|
|
Quant: "Q4_K_M"}}, modelplan.Shape{Params: params}, c.pool, 0)) {
|
|
return true
|
|
}
|
|
}
|
|
// MLX safetensors are Apple's, which no engine here reads; a repository
|
|
// tagged so may still carry a GGUF, judged above.
|
|
if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") {
|
|
// vLLM, which runs on cards only.
|
|
pool := c.pool
|
|
if m.FP8() {
|
|
pool = c.fp8
|
|
}
|
|
if !c.gpus() || len(pool) == 0 {
|
|
return false
|
|
}
|
|
b := m.WeightBytes()
|
|
if b == 0 {
|
|
if params == 0 {
|
|
return true
|
|
}
|
|
b = params * 2
|
|
}
|
|
return anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m.safetensors", Format: "safetensors",
|
|
Bytes: b}}, modelplan.Shape{Params: params}, pool, 0))
|
|
}
|
|
return false
|
|
}
|
|
|
|
func anyFits(opts []modelplan.Option) bool {
|
|
return slices.ContainsFunc(opts, modelplan.Option.Fits)
|
|
}
|
|
|
|
// note is what the listing was judged by, for the page to say.
|
|
func (c *clusterFit) note(l *Lang) string {
|
|
if !c.gpus() {
|
|
return l.T("explore.fit_cpu", c.host)
|
|
}
|
|
return l.T("explore.fit", c.card, c.cc, c.host)
|
|
}
|
|
|
|
// holds judges one file set of an open repository, whose size is known: a
|
|
// verdict that it cannot be served here, and false, when no node holds it.
|
|
func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) {
|
|
format := ""
|
|
switch lower := strings.ToLower(name); {
|
|
case strings.HasSuffix(lower, ".gguf"):
|
|
format = "gguf"
|
|
case strings.HasSuffix(lower, ".safetensors"):
|
|
format = "safetensors"
|
|
}
|
|
if format == "" || bytes <= 0 {
|
|
return gguf.Verdict{}, true
|
|
}
|
|
opts := modelplan.Options([]modelplan.Artifact{{Name: name, Format: format, Bytes: bytes,
|
|
Quant: modelplan.QuantOf(name)}}, modelplan.Shape{Params: params}, c.pool, 0)
|
|
if anyFits(opts) {
|
|
return gguf.Verdict{}, true
|
|
}
|
|
if format == "safetensors" && !c.gpus() {
|
|
return gguf.Verdict{Checked: true, Reason: l.T("card.no_gpu")}, false
|
|
}
|
|
if len(opts) == 0 {
|
|
// No width of any node's cards holds it, so none is offered: what
|
|
// the weights take against all the cards one node has.
|
|
have := 0
|
|
for _, n := range c.pool {
|
|
have = max(have, n.GPUs*n.GPUMemGB)
|
|
}
|
|
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big",
|
|
float64(bytes)/(1<<30), float64(have)*0.9)}, false
|
|
}
|
|
best := opts[0]
|
|
for _, o := range opts[1:] {
|
|
if o.HaveGB-o.NeedGB > best.HaveGB-best.NeedGB {
|
|
best = o
|
|
}
|
|
}
|
|
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false
|
|
}
|
|
|
|
// cpuSpeed is about how many tokens a second one reply gets from a GGUF of
|
|
// bytes on the fastest node whose RAM holds it, and that node; share is the
|
|
// part a token reads (ActiveShare). 0 when no measured node holds it.
|
|
func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) {
|
|
var best cpuHost
|
|
for _, n := range c.cpus {
|
|
need := float64(bytes) + float64(1<<30) + float64(bytes)/20
|
|
if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs {
|
|
best = n
|
|
}
|
|
}
|
|
if best.id == "" {
|
|
return 0, ""
|
|
}
|
|
return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id
|
|
}
|
|
|
|
// speedLabel is a rate as the page shows it: whole tokens from ten up, one
|
|
// decimal below, and under one said so.
|
|
func speedLabel(tps float64) string {
|
|
switch {
|
|
case tps <= 0:
|
|
return ""
|
|
case tps < 1:
|
|
return "<1"
|
|
case tps < 10:
|
|
return "~" + strconv.FormatFloat(tps, 'f', 1, 64)
|
|
}
|
|
return "~" + strconv.FormatFloat(tps, 'f', 0, 64)
|
|
}
|
|
|
|
// rowSpeed is a listed repository's speed on the CPU, judged at Q4_K_M:
|
|
// GGUF only, which is what runs there.
|
|
func (c *clusterFit) rowSpeed(m hf.Model) (string, string) {
|
|
if m.GGUF == nil && !m.HasTag("gguf") || m.Params() <= 0 {
|
|
return "", ""
|
|
}
|
|
b := int64(float64(m.Params()) * ggufBitsPerWeight / 8)
|
|
tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params()))
|
|
return speedLabel(tps), node
|
|
}
|