llmgw/internal/web/admin_serve.go

package web

import (
	"context"
	"errors"
	"fmt"
	"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
	"strconv"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)

// CatalogAdmin adds a model to the catalogue in force.
type CatalogAdmin interface {
	Add(ctx context.Context, m catalog.Model, by string) error
	Declared(name string) bool
	// Sources maps a weights source to the catalog models that use it.
	Sources(ctx context.Context) (map[string][]catalog.Model, error)
	// Scale changes how a model added at runtime runs: always, on demand, or
	// not at all. One declared in configuration is refused.
	Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error)
	// SetClass, SetOwner and Resources change what a model's capacity is
	// worth, whose quota it counts against, and what a replica reserves.
	SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
	SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
	Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
	SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
}

// serveForm is the offer to put one mirrored version into service.
//
// Everything a catalogue entry needs that the mirror already knows is filled
// in: the name, the source, and the engine that reads that kind of file. What
// is left is what nobody can infer — the context pool and how many replicas —
// and those are fields rather than guesses presented as facts.
type serveForm struct {
	Model   string
	Version string
	Source  string
	Engine  string
	// Profile is the profile name, which doubles as how a node_selector would
	// be written later. Named for the hardware rather than the model, because
	// a profile is about where a model runs. Derived from the placement, not
	// typed: a name is only a name, and a field for it read as though calling
	// a profile "gpu" would put it on one.
	Profile string
	// Placements are where it can run: each GPU class the fleet has, and for
	// llama.cpp the CPU. One is selected.
	Placements []placeChoice
	// GPUs is how many cards a replica takes, when placed on a GPU class.
	GPUs int
	// Parallel is how many requests one replica serves at once.
	Parallel int
	// FreeGPUs is how many cards no replica holds, across the fleet: what
	// choosing the CPU leaves unused.
	FreeGPUs int
	// Files is how many files the version holds, above one for a set.
	Files int
	// MinMemoryGB is the host RAM to ask for, estimated from the weights.
	//
	// Filled in rather than left blank. Blank writes no constraint at all, and
	// a CPU profile has no card to be short of, so every ready node becomes a
	// candidate: a 4.7 GB model went onto a 7.9 GB machine and left it with
	// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
	// told what the model needed.
	MinMemoryGB int
	// CPUCores is the cores a replica computes on: half the smallest node it
	// fits on, or zero when no node says how many it has.
	CPUCores int
	// MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
	MemoryBasis string
	// CtxPool is the engine context pool the form starts with, and the one
	// the memory figure assumes.
	CtxPool int

	// Refusal is set when this version cannot be served as it stands, and says
	// why. Offered and then failing at placement is worse than not offered.
	Refusal string
	// Warning is set when it can be served but probably will not run.
	Warning string
	// Already is true when a model of this name is in the catalogue.
	Already bool
}

// planServe works out what serving this version would mean.
func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
	fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
) serveForm {
	var header gguf.Header
	f := serveForm{
		Model:   st.Model,
		Version: st.Version,
		Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}.
			URL(base),
		Engine:  engineFor(st.File),
		Profile: "cpu",
	}

	// A version of several files is served as the version, not as one of them.
	// The source is the directory — a trailing slash — and the agent fetches
	// every file in it and hands the engine the folder.
	if len(files) > 1 {
		f.Source = strings.TrimSuffix(
			mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/"
		f.Files = len(files)
		// From whichever file says, because a version is a mixture only when
		// somebody has put one there by hand. The weights decide the engine.
		for _, one := range files {
			if e := engineFor(one.File); e != "" {
				f.Engine = e
				break
			}
		}
		var total int64
		for _, one := range files {
			total += one.BytesDone
		}
		st.BytesTotal = total
	}

	switch f.Engine {
	case "":
		f.Refusal = l.T("serve.refuse.no_engine", extOf(st.File))
		return f
	case "llamacpp":
		f.Profile = "cpu"
		if check != nil {
			v := check(f.Source)
			if !v.Servable {
				f.Refusal = l.T("serve.refuse.llamacpp", v.Reason)
				return f
			}
			header = v.Header
		}
	default:
		f.Profile = "gpu"
	}
	f.GPUs, f.Parallel = 1, defaultParallel
	f.Placements, f.FreeGPUs = placementsFor(l, fleet, f.Engine)
	for _, pc := range f.Placements {
		if pc.Selected {
			f.Profile = pc.profile()
		}
	}
	// Whether any node can actually run it. Agents report their installed
	// engines now, so this is a fact rather than the guess it started as —
	// which was cards, and would have passed a GPU node with no vLLM on it.
	//
	// A warning, not a refusal: an engine can be installed this afternoon, and
	// a node that reports nothing is an older agent rather than an empty one.
	if !runsAnywhere(fleet, f.Engine) {
		f.Warning = l.T("serve.warn.not_installed", f.Engine, extOf(st.File))
		if f.Engine != "llamacpp" && gpus(fleet) == 0 {
			f.Warning += ". " + l.T("serve.warn.no_gpu")
		}
	}
	f.CtxPool = defaultCtxPool
	f.MinMemoryGB, f.MemoryBasis = memoryFor(st.BytesTotal, st.BytesDone), l.T("serve.basis.estimate")
	weights := st.BytesTotal
	if weights <= 0 {
		weights = st.BytesDone
	}
	if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 {
		kvHeads := header.HeadCountKV
		if kvHeads == 0 {
			kvHeads = header.HeadCount
		}
		// From the model's own shape: its KV cache for the pool the form
		// starts with, which ranges from under half a gigabyte to over five
		// across the models on the live mirror at the same 16k — a spread the
		// flat "half again" could not see.
		f.MinMemoryGB = int((b + 1<<30 - 1) >> 30)
		f.MemoryBasis = l.T("serve.basis.shape",
			l.Num(float64(weights)/(1<<30), 1), l.Num(float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30), 1),
			l.Int(defaultCtxPool), header.BlockCount, kvHeads,
			l.Num(float64(int64(1<<30)+weights/20)/(1<<30), 1))
	}
	if fleet != nil {
		var cpus []int
		for _, n := range fleet.Nodes() {
			if n.State == "ready" && (n.MemoryGB == 0 || n.MemoryGB >= f.MinMemoryGB) {
				cpus = append(cpus, n.CPUCount)
			}
		}
		f.CPUCores = catalog.SuggestCores(cpus)
	}
	if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
		big := l.T("serve.warn.too_big", biggest, f.MinMemoryGB)
		if f.Warning != "" {
			big = l.T("serve.warn.and", strings.TrimSuffix(f.Warning, "."), big)
		}
		f.Warning = big
	}
	if cat != nil && cat.Declared(st.Model) {
		f.Already = true
		f.Refusal = l.T("serve.refuse.declared", st.Model)
	}
	return f
}

// engineFor is the engine that reads this kind of weights file.
func engineFor(file string) string {
	switch strings.ToLower(extOf(file)) {
	case ".gguf":
		return "llamacpp"
	case ".safetensors", ".bin":
		return "vllm"
	}
	return ""
}

func extOf(file string) string {
	if i := strings.LastIndex(file, "."); i >= 0 {
		return file[i:]
	}
	return file
}

// runsAnywhere reports whether some node has this engine.
//
// True when nothing is known, so a fleet of agents older than the engine probe
// is not told every model is unservable. That matches what placement does with
// the same absence, and the two disagreeing would be worse than either.
func runsAnywhere(fleet Source, engine string) bool {
	if fleet == nil || engine == "" {
		return true
	}
	anyReported := false
	for _, n := range fleet.Nodes() {
		if len(n.Engines) == 0 {
			continue
		}
		anyReported = true
		for _, e := range n.Engines {
			if e == engine {
				return true
			}
		}
	}
	return !anyReported
}

func gpus(fleet Source) int {
	if fleet == nil {
		return 0
	}
	n := 0
	for _, node := range fleet.Nodes() {
		n += node.GPUTotal
	}
	return n
}

// buildModel turns the submitted form into a catalogue entry. Its refusals
// are in l, the language of whoever filled the form in.
func buildModel(l *Lang, get func(string) string) (catalog.Model, error) {
	m := catalog.Model{
		Name:           strings.TrimSpace(get("name")),
		Description:    strings.TrimSpace(get("description")),
		Source:         strings.TrimSpace(get("source")),
		RoutingProfile: "chat",
	}
	if m.Name == "" || m.Source == "" {
		return m, errors.New(l.T("serve.err.name_source"))
	}
	for _, a := range strings.Split(get("aliases"), ",") {
		if a = strings.TrimSpace(a); a != "" {
			m.Aliases = append(m.Aliases, a)
		}
	}

	ctxLen, err := positive(l, get("context_length"), "serve.what.context_length")
	if err != nil {
		return m, err
	}
	m.ContextLength = ctxLen
	if v := strings.TrimSpace(get("max_output")); v != "" {
		if m.MaxOutput, err = positive(l, v, "serve.what.max_output"); err != nil {
			return m, err
		}
	}

	min, err := positive(l, get("replicas"), "serve.what.replicas")
	if err != nil {
		return m, err
	}
	m.Scaling = scalingFor(get("mode"), min, 0)

	engine := strings.TrimSpace(get("engine"))
	if engine == "" {
		return m, errors.New(l.T("serve.err.engine"))
	}
	p := catalog.Profile{Engine: engine, Weight: 1, Priority: 100}
	gpus := 1
	if v := strings.TrimSpace(get("gpus")); v != "" {
		if gpus, err = positive(l, v, "serve.what.gpus"); err != nil {
			return m, err
		}
		gpus = max(gpus, 1)
	}
	place := placeChoice{Value: strings.TrimSpace(get("placement"))}
	switch class, onGPU := strings.CutPrefix(place.Value, "gpu:"); {
	case onGPU && class != "":
		// Placement is decided by gpu_class, which is what the selector
		// sets; the count is gpus_per_node for llama.cpp, which spreads
		// layers over a node's cards, and tp for the others.
		p.GPUClass = class
		if engine == "llamacpp" {
			p.GPUsPerNode = gpus
		} else {
			p.TP = gpus
		}
		place.GPUs = gpus
	case place.Value == "" || place.Value == "cpu":
		if engine != "llamacpp" {
			return m, errors.New(l.T("serve.err.needs_gpu", engine))
		}
		place.Value = "cpu"
	default:
		return m, errors.New(l.T("serve.err.placement", place.Value))
	}
	p.Name = place.profile()
	if v := strings.TrimSpace(get("profile")); v != "" {
		p.Name = v
	}
	// The pool, not the per-request limit: llama.cpp divides it across its
	// parallel slots, so the number a client may use is smaller and is what
	// context_length above says.
	if v := strings.TrimSpace(get("ctx_size")); v != "" {
		if _, err := positive(l, v, "serve.what.ctx_pool"); err != nil {
			return m, err
		}
		p.Args = []string{"--ctx-size", v}
	}
	if v := strings.TrimSpace(get("parallel")); v != "" && engine == "llamacpp" {
		n, err := positive(l, v, "serve.what.parallel")
		if err != nil {
			return m, err
		}
		if n > 0 {
			p.Args = append(p.Args, "--parallel", strconv.Itoa(n))
		}
	}
	if v := strings.TrimSpace(get("min_memory_gb")); v != "" {
		if p.MinMemoryGB, err = positive(l, v, "serve.what.memory"); err != nil {
			return m, err
		}
	}
	if v := strings.TrimSpace(get("cpu_cores")); v != "" {
		if p.CPUCores, err = positive(l, v, "serve.what.cores"); err != nil {
			return m, err
		}
	}
	if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" {
		// What keeps a large model off the small nodes. A CPU profile has no
		// card to be short of, so without this every ready node looks like a
		// candidate and the first one picked is the one that gets OOM-killed.
		p.NodeSelector = map[string]string{k: v}
	}
	m.Profiles = []catalog.Profile{p}
	return m, nil
}

// positive reads a whole number from a form field; what is the key naming
// the field, for the refusal.
func positive(l *Lang, v, what string) (int, error) {
	n, err := strconv.Atoi(strings.TrimSpace(v))
	if err != nil || n < 0 {
		return 0, errors.New(l.T("serve.err.whole_number", l.T(what)))
	}
	return n, nil
}

func init() {
	registerKeys("serve.what.context_length", "serve.what.max_output", "serve.what.replicas",
		"serve.what.gpus", "serve.what.ctx_pool", "serve.what.parallel", "serve.what.memory",
		"serve.what.cores")
}

// memoryFor estimates the host RAM a model needs, from the size of its weights.
//
// Weights plus half again, plus two. The half covers the KV pool, which grows
// with the same things the weights do — layers and heads — and the two are the
// engine's own footprint and the page cache it will not get back. It is an
// estimate and the field it fills is editable; what it is not is blank, which
// is what put a 4.7 GB model on a 7.9 GB node.
//
// The honest number comes from running it: `llmgw ctl bench` measures what a
// replica can actually do, and that is a different question from what it needs
// to start.
func memoryFor(total, done int64) int {
	size := total
	if size <= 0 {
		size = done
	}
	if size <= 0 {
		return 0
	}
	gb := float64(size) / (1 << 30)
	return int(gb*1.5) + 2
}

// biggestNode is the most memory any single node reports, or 0 when none says.
func biggestNode(fleet Source) int {
	if fleet == nil {
		return 0
	}
	most := 0
	for _, n := range fleet.Nodes() {
		if n.MemoryGB > most {
			most = n.MemoryGB
		}
	}
	return most
}

// Run modes a page offers for a model, in the words a person would use.
const (
	modeAlways   = "always"
	modeOnDemand = "ondemand"
	modeStopped  = "stopped"
)

// defaultIdle is how long an on-demand model set from a page stays loaded
// after its last request.
const defaultIdle = 15 * time.Minute

// scalingFor turns a run mode into scaling: replicas is how many run, or may
// run on demand; idle keeps an on-demand model's timeout when it has one.
func scalingFor(mode string, replicas int, idle time.Duration) catalog.Scaling {
	if replicas < 1 {
		replicas = 1
	}
	switch mode {
	case modeOnDemand:
		if idle <= 0 {
			idle = defaultIdle
		}
		return catalog.Scaling{Min: 0, Max: replicas, IdleTimeout: idle}
	case modeStopped:
		return catalog.Scaling{Min: 0, Max: 0}
	default:
		return catalog.Scaling{Min: replicas, Max: replicas}
	}
}

// modeOf names how a model runs.
func modeOf(sc catalog.Scaling) string {
	switch {
	case sc.Min > 0:
		return modeAlways
	case sc.Max > 0:
		return modeOnDemand
	default:
		return modeStopped
	}
}

// defaultCtxPool is the llama.cpp context pool the serve form starts with.
// --parallel divides it among slots; it does not multiply it.
const defaultCtxPool = 16384

// defaultParallel is the requests a llama.cpp replica serves at once when
// the form is not changed: the agent's own default.
const defaultParallel = 4

// placeChoice is one place a model can run: a GPU class, or the CPU.
type placeChoice struct {
	// Value is "gpu:<class>" or "cpu", as the form submits it.
	Value string
	Label string
	// GPUs is how many cards a replica takes, for naming the profile.
	GPUs     int
	Selected bool
}

// profile names the profile for where it runs: the class, with the count when
// it is more than one card ("t4", "t4-x4"), or "cpu".
func (c placeChoice) profile() string {
	class, ok := strings.CutPrefix(c.Value, "gpu:")
	if !ok || class == "" {
		return "cpu"
	}
	if c.GPUs > 1 {
		return fmt.Sprintf("%s-x%d", class, c.GPUs)
	}
	return class
}

// placementsFor lists where engine can run on this fleet, with one selected,
// and how many GPUs are free.
//
// A GPU when the fleet has one: a GGUF model was put on the CPU by this form
// on a cluster of GPUs, ran at a tenth of their speed, and nothing on the way
// said why — the form had no choice to offer and named the profile "cpu". The
// class selected is the one with the most cards free.
func placementsFor(l *Lang, fleet Source, engine string) ([]placeChoice, int) {
	var out []placeChoice
	free, best := 0, -1
	if fleet != nil {
		for _, c := range fleet.Summary().GPUs {
			n := c.Free()
			free += n
			out = append(out, placeChoice{
				Value: "gpu:" + c.Class,
				Label: l.T("serve.place.gpu", c.Class, n, c.Total),
			})
			if best < 0 || n > fleet.Summary().GPUs[best].Free() {
				best = len(out) - 1
			}
		}
	}
	if best >= 0 {
		out[best].Selected = true
	}
	if engine == "llamacpp" {
		label := l.T("serve.place.cpu_slower")
		if len(out) == 0 {
			label = l.T("serve.place.cpu_only")
		}
		out = append(out, placeChoice{Value: "cpu", Label: label, Selected: best < 0})
	}
	return out, free
}

// freeGPUs is how many cards no replica holds across the fleet.
func freeGPUs(fleet Source) int {
	if fleet == nil {
		return 0
	}
	n := 0
	for _, c := range fleet.Summary().GPUs {
		n += c.Free()
	}
	return n
}

// describeServe says, in one line, what a model will run as: the summary
// shown before it is added and in the message after. The form's script says
// the same with the same sentences, so the two cannot drift apart in wording.
func describeServe(l *Lang, m catalog.Model) string {
	if len(m.Profiles) == 0 {
		return ""
	}
	p := m.Profiles[0]
	engine := map[string]string{"llamacpp": "llama.cpp", "vllm": "vLLM", "sglang": "SGLang", "trtllm": "TensorRT-LLM"}[p.Engine]
	if engine == "" {
		engine = p.Engine
	}
	where := l.T("serve.on_cpu", engine)
	if p.GPUClass != "" {
		n := max(p.GPUsPerNode, p.TP, 1)
		where = l.T("serve.on_gpu_one", engine, p.GPUClass)
		if n > 1 {
			where = l.T("serve.on_gpus", engine, n, p.GPUClass)
		}
	}
	parts := []string{where}
	if p.MinMemoryGB > 0 {
		parts = append(parts, l.T("serve.host_memory", l.Int(int64(p.MinMemoryGB))))
	}
	if p.GPUClass == "" && p.CPUCores > 0 {
		parts = append(parts, l.N("serve.cores", int64(p.CPUCores)))
	}
	pool, parallel := argInt(p.Args, "--ctx-size"), argInt(p.Args, "--parallel")
	switch {
	case pool > 0 && parallel > 0:
		parts = append(parts, l.T("serve.pool_parallel", l.Int(int64(pool)), l.Int(int64(parallel)), l.Int(int64(pool/parallel))))
	case pool > 0:
		parts = append(parts, l.T("serve.pool", l.Int(int64(pool))))
	}
	return strings.Join(parts, ", ")
}

// argInt is the number after flag in args, or 0.
func argInt(args []string, flag string) int {
	for i := 0; i+1 < len(args); i++ {
		if args[i] == flag {
			n, _ := strconv.Atoi(args[i+1])
			return n
		}
	}
	return 0
}

// asJudge makes a model built from the serve form a judge when its weights
// are one's — Llama Guard, Qwen3Guard, ShieldGemma: role guard, that judge's
// format, under the name the policies give their judge when they give one
// and the form gave another. It says whether it did, and the name the form
// gave when it was changed.
func (p *AccountPages) asJudge(m *catalog.Model) (judge bool, renamed string) {
	format, ok := guard.FormatOfWeights(m.Source)
	if !ok {
		return false, ""
	}
	m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: format}, ""
	if p.guardLibrary == nil {
		return true, ""
	}
	asked := map[string]bool{}
	for _, pol := range p.guardLibrary().Policies {
		for _, j := range pol.Judge.Judges() {
			asked[j] = true
		}
	}
	if asked[m.Name] || len(asked) != 1 {
		return true, ""
	}
	for name := range asked {
		renamed, m.Name = m.Name, name
	}
	return true, renamed
}