Judged by the nodes as they are now, through modelplan.Options, the planner that serving a model uses: GGUF at Q4_K_M from the parameter count, safetensors at the size the registry's dtype breakdown gives (packed AWQ, GPTQ and MXFP4 at four bits), FP8 only where a card has compute capability 8.9, MLX safetensors never. A picture model is judged against one card. A cluster with cards asks the registry for safetensors too, a hundred at a time; one without asks for GGUF only. The page says what it judged by: the largest card, its capability, the most RAM on one node. On a repository's page a file set no node holds is marked with what it needs, against what the best node offers, and has no Fetch button. Tried against the PoC's seven nodes and Hugging Face's trending hundred: 63 shown, the rest too big for one 24 GB Turing card or FP8. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
182 lines
5.7 KiB
Go
182 lines
5.7 KiB
Go
package web
|
|
|
|
import (
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
|
)
|
|
|
|
// The explorer lists what this cluster can run, judged by the nodes it has
|
|
// now rather than by a size picked from a list: a site with H100s sees the
|
|
// 70B models, one with a T4 and CPUs sees what fits there. From the
|
|
// registry's listing alone — a parameter count, safetensors by dtype, tags
|
|
// — so no repository is opened to decide; the repository's page sizes each
|
|
// file exactly.
|
|
|
|
// clusterFit is the hardware a listing is judged against.
|
|
type clusterFit struct {
|
|
pool []modelplan.PoolNode
|
|
// fp8 are the nodes with a card vLLM runs FP8 weights on, compute
|
|
// capability 8.9 or newer, as the agent requires.
|
|
fp8 []modelplan.PoolNode
|
|
// card is the largest card's memory in GB, cc its compute capability;
|
|
// host the most memory one node has.
|
|
card int
|
|
cc string
|
|
host int
|
|
}
|
|
|
|
// fp8Capability is the oldest card vLLM runs FP8 weights on.
|
|
const fp8Capability = 8.9
|
|
|
|
// fitOf is the cluster's hardware now, nil when there is none to judge by:
|
|
// no fleet to ask, or no node ready.
|
|
func (p *AccountPages) fitOf() *clusterFit {
|
|
if p.fleet == nil {
|
|
return nil
|
|
}
|
|
nodes := p.fleet.Nodes()
|
|
c := &clusterFit{pool: adminapi.PoolNodesOf(nodes)}
|
|
if len(c.pool) == 0 {
|
|
return nil
|
|
}
|
|
newest := map[string]float64{}
|
|
for _, n := range nodes {
|
|
for _, s := range n.ComputeCapabilities {
|
|
if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] {
|
|
newest[n.ID] = v
|
|
}
|
|
}
|
|
}
|
|
for _, n := range c.pool {
|
|
c.host = max(c.host, n.HostMemGB)
|
|
if n.GPUs > 0 && n.GPUMemGB >= c.card {
|
|
if n.GPUMemGB > c.card || newest[n.ID] > atof(c.cc) {
|
|
c.cc = strconv.FormatFloat(newest[n.ID], 'f', 1, 64)
|
|
}
|
|
c.card = n.GPUMemGB
|
|
}
|
|
if newest[n.ID] >= fp8Capability {
|
|
c.fp8 = append(c.fp8, n)
|
|
}
|
|
}
|
|
return c
|
|
}
|
|
|
|
func atof(s string) float64 { v, _ := strconv.ParseFloat(s, 64); return v }
|
|
|
|
// gpus reports whether any node has a card.
|
|
func (c *clusterFit) gpus() bool { return c.card > 0 }
|
|
|
|
// pictureTasks are a pipeline ComfyUI runs, on a card, rather than an
|
|
// engine for text.
|
|
var pictureTasks = []string{"text-to-image", "image-to-image", "image-to-3d", "text-to-3d",
|
|
"text-to-video", "image-to-video", "unconditional-image-generation"}
|
|
|
|
// ggufBitsPerWeight is the quantisation a GGUF repository is judged by:
|
|
// Q4_K_M, the smallest of those published that answers as well as most
|
|
// people want. A repository of Q8 only is judged kindly; its page says.
|
|
const ggufBitsPerWeight = 4.8
|
|
|
|
// runs reports whether some node here could hold the repository's model.
|
|
func (c *clusterFit) runs(m hf.Model) bool {
|
|
params := m.Params()
|
|
if slices.Contains(pictureTasks, m.PipelineTag) || m.HasTag("diffusers") {
|
|
b := m.WeightBytes()
|
|
if b == 0 {
|
|
b = params * 2
|
|
}
|
|
// ComfyUI holds the model on one card, with room for its working.
|
|
return c.gpus() && float64(b+2<<30) <= float64(c.card)*0.9*(1<<30)
|
|
}
|
|
if m.GGUF != nil || m.HasTag("gguf") && m.Safetensors == nil {
|
|
if params == 0 {
|
|
return true
|
|
}
|
|
b := int64(float64(params) * ggufBitsPerWeight / 8)
|
|
if anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b,
|
|
Quant: "Q4_K_M"}}, modelplan.Shape{Params: params}, c.pool, 0)) {
|
|
return true
|
|
}
|
|
}
|
|
// MLX safetensors are Apple's, which no engine here reads; a repository
|
|
// tagged so may still carry a GGUF, judged above.
|
|
if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") {
|
|
// vLLM, which runs on cards only.
|
|
pool := c.pool
|
|
if m.FP8() {
|
|
pool = c.fp8
|
|
}
|
|
if !c.gpus() || len(pool) == 0 {
|
|
return false
|
|
}
|
|
b := m.WeightBytes()
|
|
if b == 0 {
|
|
if params == 0 {
|
|
return true
|
|
}
|
|
b = params * 2
|
|
}
|
|
return anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m.safetensors", Format: "safetensors",
|
|
Bytes: b}}, modelplan.Shape{Params: params}, pool, 0))
|
|
}
|
|
return false
|
|
}
|
|
|
|
func anyFits(opts []modelplan.Option) bool {
|
|
return slices.ContainsFunc(opts, modelplan.Option.Fits)
|
|
}
|
|
|
|
// note is what the listing was judged by, for the page to say.
|
|
func (c *clusterFit) note(l *Lang) string {
|
|
if !c.gpus() {
|
|
return l.T("explore.fit_cpu", c.host)
|
|
}
|
|
return l.T("explore.fit", c.card, c.cc, c.host)
|
|
}
|
|
|
|
// holds judges one file set of an open repository, whose size is known: a
|
|
// verdict that it cannot be served here, and false, when no node holds it.
|
|
func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) {
|
|
format := ""
|
|
switch lower := strings.ToLower(name); {
|
|
case strings.HasSuffix(lower, ".gguf"):
|
|
format = "gguf"
|
|
case strings.HasSuffix(lower, ".safetensors"):
|
|
format = "safetensors"
|
|
}
|
|
if format == "" || bytes <= 0 {
|
|
return gguf.Verdict{}, true
|
|
}
|
|
opts := modelplan.Options([]modelplan.Artifact{{Name: name, Format: format, Bytes: bytes,
|
|
Quant: modelplan.QuantOf(name)}}, modelplan.Shape{Params: params}, c.pool, 0)
|
|
if anyFits(opts) {
|
|
return gguf.Verdict{}, true
|
|
}
|
|
if format == "safetensors" && !c.gpus() {
|
|
return gguf.Verdict{Checked: true, Reason: l.T("card.no_gpu")}, false
|
|
}
|
|
if len(opts) == 0 {
|
|
// No width of any node's cards holds it, so none is offered: what
|
|
// the weights take against all the cards one node has.
|
|
have := 0
|
|
for _, n := range c.pool {
|
|
have = max(have, n.GPUs*n.GPUMemGB)
|
|
}
|
|
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big",
|
|
float64(bytes)/(1<<30), float64(have)*0.9)}, false
|
|
}
|
|
best := opts[0]
|
|
for _, o := range opts[1:] {
|
|
if o.HaveGB-o.NeedGB > best.HaveGB-best.NeedGB {
|
|
best = o
|
|
}
|
|
}
|
|
return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false
|
|
}
|