Files
llmgw/internal/web/explore_fit.go
T
Ilker ManapandClaude Opus 5.5 84444e71c8
CI / test (push) Failing after 6m37s
How fast a model would answer on a CPU, before it is fetched
Each agent measures its node's memory read rate when it starts: every CPU
sums a share of a buffer past any cache, a tenth of free memory at most,
the best of four passes, in a fraction of a second; reported in Host
(mem_bandwidth_gbs) and NodeView. A token on a CPU reads every weight it
uses once, so one reply's speed is about that rate over those bytes:
modelplan.CPUTokensPerSecond, 0.68 of the measured rate plus 5 ms a token,
calibrated on the PoC against llama.cpp's predicted_per_second (node01
24.9 GB/s, qwen2.5-1.5b: 14.5 measured, 14.7 estimated; node05 41.9 GB/s,
qwen2.5-14b: 3.3 and 3.2). A model of experts named …-A3B reads its active
part (ActiveShare).

The explorer shows it as CPU tok/s, at Q4_K_M on the fastest node that
holds the model; a repository's page gives it for each GGUF by its size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 00:09:47 +01:00

237 lines
7.4 KiB
Go

package web
import (
"slices"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
// The explorer lists what this cluster can run, judged by the nodes it has
// now rather than by a size picked from a list: a site with H100s sees the
// 70B models, one with a T4 and CPUs sees what fits there. From the
// registry's listing alone — a parameter count, safetensors by dtype, tags
// — so no repository is opened to decide; the repository's page sizes each
// file exactly.
// clusterFit is the hardware a listing is judged against.
type clusterFit struct {
pool []modelplan.PoolNode
// fp8 are the nodes with a card vLLM runs FP8 weights on, compute
// capability 8.9 or newer, as the agent requires.
fp8 []modelplan.PoolNode
// card is the largest card's memory in GB, cc its compute capability;
// host the most memory one node has.
card int
cc string
host int
// cpus are the nodes whose memory's speed their agent measured, for
// how fast a model would answer on one.
cpus []cpuHost
}
// cpuHost is a node as a model on its CPU sees it.
type cpuHost struct {
id string
ramGB int
bwGBs float64
}
// fp8Capability is the oldest card vLLM runs FP8 weights on.
const fp8Capability = 8.9
// fitOf is the cluster's hardware now, nil when there is none to judge by:
// no fleet to ask, or no node ready.
func (p *AccountPages) fitOf() *clusterFit {
if p.fleet == nil {
return nil
}
nodes := p.fleet.Nodes()
c := &clusterFit{pool: adminapi.PoolNodesOf(nodes)}
if len(c.pool) == 0 {
return nil
}
newest := map[string]float64{}
for _, n := range nodes {
if n.State == "ready" && n.MemBandwidthGBs > 0 {
c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
}
for _, s := range n.ComputeCapabilities {
if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] {
newest[n.ID] = v
}
}
}
for _, n := range c.pool {
c.host = max(c.host, n.HostMemGB)
if n.GPUs > 0 && n.GPUMemGB >= c.card {
if n.GPUMemGB > c.card || newest[n.ID] > atof(c.cc) {
c.cc = strconv.FormatFloat(newest[n.ID], 'f', 1, 64)
}
c.card = n.GPUMemGB
}
if newest[n.ID] >= fp8Capability {
c.fp8 = append(c.fp8, n)
}
}
return c
}
func atof(s string) float64 { v, _ := strconv.ParseFloat(s, 64); return v }
// gpus reports whether any node has a card.
func (c *clusterFit) gpus() bool { return c.card > 0 }
// pictureTasks are a pipeline ComfyUI runs, on a card, rather than an
// engine for text.
var pictureTasks = []string{"text-to-image", "image-to-image", "image-to-3d", "text-to-3d",
"text-to-video", "image-to-video", "unconditional-image-generation"}
// ggufBitsPerWeight is the quantisation a GGUF repository is judged by:
// Q4_K_M, the smallest of those published that answers as well as most
// people want. A repository of Q8 only is judged kindly; its page says.
const ggufBitsPerWeight = 4.8
// runs reports whether some node here could hold the repository's model.
func (c *clusterFit) runs(m hf.Model) bool {
params := m.Params()
if slices.Contains(pictureTasks, m.PipelineTag) || m.HasTag("diffusers") {
b := m.WeightBytes()
if b == 0 {
b = params * 2
}
// ComfyUI holds the model on one card, with room for its working.
return c.gpus() && float64(b+2<<30) <= float64(c.card)*0.9*(1<<30)
}
if m.GGUF != nil || m.HasTag("gguf") && m.Safetensors == nil {
if params == 0 {
return true
}
b := int64(float64(params) * ggufBitsPerWeight / 8)
if anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b,
Quant: "Q4_K_M"}}, modelplan.Shape{Params: params}, c.pool, 0)) {
return true
}
}
// MLX safetensors are Apple's, which no engine here reads; a repository
// tagged so may still carry a GGUF, judged above.
if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") {
// vLLM, which runs on cards only.
pool := c.pool
if m.FP8() {
pool = c.fp8
}
if !c.gpus() || len(pool) == 0 {
return false
}
b := m.WeightBytes()
if b == 0 {
if params == 0 {
return true
}
b = params * 2
}
return anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m.safetensors", Format: "safetensors",
Bytes: b}}, modelplan.Shape{Params: params}, pool, 0))
}
return false
}
func anyFits(opts []modelplan.Option) bool {
return slices.ContainsFunc(opts, modelplan.Option.Fits)
}
// note is what the listing was judged by, for the page to say.
func (c *clusterFit) note(l *Lang) string {
if !c.gpus() {
return l.T("explore.fit_cpu", c.host)
}
return l.T("explore.fit", c.card, c.cc, c.host)
}
// holds judges one file set of an open repository, whose size is known: a
// verdict that it cannot be served here, and false, when no node holds it.
func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) {
format := ""
switch lower := strings.ToLower(name); {
case strings.HasSuffix(lower, ".gguf"):
format = "gguf"
case strings.HasSuffix(lower, ".safetensors"):
format = "safetensors"
}
if format == "" || bytes <= 0 {
return gguf.Verdict{}, true
}
opts := modelplan.Options([]modelplan.Artifact{{Name: name, Format: format, Bytes: bytes,
Quant: modelplan.QuantOf(name)}}, modelplan.Shape{Params: params}, c.pool, 0)
if anyFits(opts) {
return gguf.Verdict{}, true
}
if format == "safetensors" && !c.gpus() {
return gguf.Verdict{Checked: true, Reason: l.T("card.no_gpu")}, false
}
if len(opts) == 0 {
// No width of any node's cards holds it, so none is offered: what
// the weights take against all the cards one node has.
have := 0
for _, n := range c.pool {
have = max(have, n.GPUs*n.GPUMemGB)
}
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big",
float64(bytes)/(1<<30), float64(have)*0.9)}, false
}
best := opts[0]
for _, o := range opts[1:] {
if o.HaveGB-o.NeedGB > best.HaveGB-best.NeedGB {
best = o
}
}
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false
}
// cpuSpeed is about how many tokens a second one reply gets from a GGUF of
// bytes on the fastest node whose RAM holds it, and that node; share is the
// part a token reads (ActiveShare). 0 when no measured node holds it.
func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) {
var best cpuHost
for _, n := range c.cpus {
need := float64(bytes) + float64(1<<30) + float64(bytes)/20
if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs {
best = n
}
}
if best.id == "" {
return 0, ""
}
return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id
}
// speedLabel is a rate as the page shows it: whole tokens from ten up, one
// decimal below, and under one said so.
func speedLabel(tps float64) string {
switch {
case tps <= 0:
return ""
case tps < 1:
return "<1"
case tps < 10:
return "~" + strconv.FormatFloat(tps, 'f', 1, 64)
}
return "~" + strconv.FormatFloat(tps, 'f', 0, 64)
}
// rowSpeed is a listed repository's speed on the CPU, judged at Q4_K_M:
// GGUF only, which is what runs there.
func (c *clusterFit) rowSpeed(m hf.Model) (string, string) {
if m.GGUF == nil && !m.HasTag("gguf") || m.Params() <= 0 {
return "", ""
}
b := int64(float64(m.Params()) * ggufBitsPerWeight / 8)
tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params()))
return speedLabel(tps), node
}