package web import ( "slices" "strconv" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // The explorer lists what this cluster can run, judged by the nodes it has // now rather than by a size picked from a list: a site with H100s sees the // 70B models, one with a T4 and CPUs sees what fits there. From the // registry's listing alone — a parameter count, safetensors by dtype, tags // — so no repository is opened to decide; the repository's page sizes each // file exactly. // clusterFit is the hardware a listing is judged against. type clusterFit struct { pool []modelplan.PoolNode // fp8 are the nodes with a card vLLM runs FP8 weights on, compute // capability 8.9 or newer, as the agent requires. fp8 []modelplan.PoolNode // card is the largest card's memory in GB, cc its compute capability; // host the most memory one node has. card int cc string host int } // fp8Capability is the oldest card vLLM runs FP8 weights on. const fp8Capability = 8.9 // fitOf is the cluster's hardware now, nil when there is none to judge by: // no fleet to ask, or no node ready. func (p *AccountPages) fitOf() *clusterFit { if p.fleet == nil { return nil } nodes := p.fleet.Nodes() c := &clusterFit{pool: adminapi.PoolNodesOf(nodes)} if len(c.pool) == 0 { return nil } newest := map[string]float64{} for _, n := range nodes { for _, s := range n.ComputeCapabilities { if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] { newest[n.ID] = v } } } for _, n := range c.pool { c.host = max(c.host, n.HostMemGB) if n.GPUs > 0 && n.GPUMemGB >= c.card { if n.GPUMemGB > c.card || newest[n.ID] > atof(c.cc) { c.cc = strconv.FormatFloat(newest[n.ID], 'f', 1, 64) } c.card = n.GPUMemGB } if newest[n.ID] >= fp8Capability { c.fp8 = append(c.fp8, n) } } return c } func atof(s string) float64 { v, _ := strconv.ParseFloat(s, 64); return v } // gpus reports whether any node has a card. func (c *clusterFit) gpus() bool { return c.card > 0 } // pictureTasks are a pipeline ComfyUI runs, on a card, rather than an // engine for text. var pictureTasks = []string{"text-to-image", "image-to-image", "image-to-3d", "text-to-3d", "text-to-video", "image-to-video", "unconditional-image-generation"} // ggufBitsPerWeight is the quantisation a GGUF repository is judged by: // Q4_K_M, the smallest of those published that answers as well as most // people want. A repository of Q8 only is judged kindly; its page says. const ggufBitsPerWeight = 4.8 // runs reports whether some node here could hold the repository's model. func (c *clusterFit) runs(m hf.Model) bool { params := m.Params() if slices.Contains(pictureTasks, m.PipelineTag) || m.HasTag("diffusers") { b := m.WeightBytes() if b == 0 { b = params * 2 } // ComfyUI holds the model on one card, with room for its working. return c.gpus() && float64(b+2<<30) <= float64(c.card)*0.9*(1<<30) } if m.GGUF != nil || m.HasTag("gguf") && m.Safetensors == nil { if params == 0 { return true } b := int64(float64(params) * ggufBitsPerWeight / 8) if anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: b, Quant: "Q4_K_M"}}, modelplan.Shape{Params: params}, c.pool, 0)) { return true } } // MLX safetensors are Apple's, which no engine here reads; a repository // tagged so may still carry a GGUF, judged above. if (m.Safetensors != nil || m.HasTag("safetensors")) && !m.HasTag("mlx") { // vLLM, which runs on cards only. pool := c.pool if m.FP8() { pool = c.fp8 } if !c.gpus() || len(pool) == 0 { return false } b := m.WeightBytes() if b == 0 { if params == 0 { return true } b = params * 2 } return anyFits(modelplan.Options([]modelplan.Artifact{{Name: "m.safetensors", Format: "safetensors", Bytes: b}}, modelplan.Shape{Params: params}, pool, 0)) } return false } func anyFits(opts []modelplan.Option) bool { return slices.ContainsFunc(opts, modelplan.Option.Fits) } // note is what the listing was judged by, for the page to say. func (c *clusterFit) note(l *Lang) string { if !c.gpus() { return l.T("explore.fit_cpu", c.host) } return l.T("explore.fit", c.card, c.cc, c.host) } // holds judges one file set of an open repository, whose size is known: a // verdict that it cannot be served here, and false, when no node holds it. func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verdict, bool) { format := "" switch lower := strings.ToLower(name); { case strings.HasSuffix(lower, ".gguf"): format = "gguf" case strings.HasSuffix(lower, ".safetensors"): format = "safetensors" } if format == "" || bytes <= 0 { return gguf.Verdict{}, true } opts := modelplan.Options([]modelplan.Artifact{{Name: name, Format: format, Bytes: bytes, Quant: modelplan.QuantOf(name)}}, modelplan.Shape{Params: params}, c.pool, 0) if anyFits(opts) { return gguf.Verdict{}, true } if format == "safetensors" && !c.gpus() { return gguf.Verdict{Checked: true, Reason: l.T("card.no_gpu")}, false } if len(opts) == 0 { // No width of any node's cards holds it, so none is offered: what // the weights take against all the cards one node has. have := 0 for _, n := range c.pool { have = max(have, n.GPUs*n.GPUMemGB) } return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", float64(bytes)/(1<<30), float64(have)*0.9)}, false } best := opts[0] for _, o := range opts[1:] { if o.HaveGB-o.NeedGB > best.HaveGB-best.NeedGB { best = o } } return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false }