llmgw/internal/web/mirror_settings.go

package web

import (
	"context"
	"errors"
	"net/http"
	"strconv"
	"strings"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// ModelSettings changes a served model beyond what the catalogue holds: its
// context, its cards, its replicas started again, its removal. The cluster
// view answers it where it can.
type ModelSettings interface {
	SetModelContext(ctx context.Context, name string, req modelserve.ContextRequest, by string) (modelserve.ContextChange, error)
	SetModelGPUs(ctx context.Context, name string, req modelserve.GPUsRequest, by string) (modelserve.GPUsChange, error)
	RestartModel(name string) (int, error)
	RemoveModel(ctx context.Context, name string, purge bool, by string) (adminapi.PurgeView, error)
}

// settings is the cluster view's ModelSettings, nil where it has none.
func (p *AccountPages) settings() ModelSettings {
	s, _ := p.fleet.(ModelSettings)
	return s
}

// registerModelSettings mounts what the card of a served model changes.
func (p *AccountPages) registerModelSettings(mux *http.ServeMux, m MirrorAdmin) {
	need := func(r *http.Request) (string, ModelSettings, error) {
		name := strings.TrimSpace(r.FormValue("model"))
		if name == "" {
			return "", nil, errors.New(langOf(r).T("mirror.err.choose_model"))
		}
		s := p.settings()
		if s == nil {
			return "", nil, errors.New(langOf(r).T("mirror.err.no_settings"))
		}
		return name, s, nil
	}

	mux.HandleFunc("POST /admin/mirror/context", p.mirrorAction(m,
		func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
			l := langOf(r)
			name, s, err := need(r)
			if err != nil {
				return "", err
			}
			tokens, err := optionalCount(l, r.FormValue("tokens"), "serve.context_length")
			if err != nil {
				return "", err
			}
			maxOut, err := optionalCount(l, r.FormValue("max_output"), "serve.max_output")
			if err != nil {
				return "", err
			}
			ch, err := s.SetModelContext(ctx, name, modelserve.ContextRequest{Tokens: tokens, MaxOutput: maxOut}, admin.Email)
			if err != nil {
				return "", err
			}
			return l.T("mirror.msg.context", name, l.Int(int64(ch.After)), l.Int(int64(ch.MaxOutput)), ch.Restarted), nil
		}))

	mux.HandleFunc("POST /admin/mirror/gpus", p.mirrorAction(m,
		func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
			l := langOf(r)
			name, s, err := need(r)
			if err != nil {
				return "", err
			}
			n, err := strconv.Atoi(strings.TrimSpace(r.FormValue("gpus")))
			if err != nil || n < 1 {
				return "", errors.New(l.T("mirror.err.gpus"))
			}
			ch, err := s.SetModelGPUs(ctx, name, modelserve.GPUsRequest{GPUs: n, Profile: r.FormValue("profile")}, admin.Email)
			if err != nil {
				return "", err
			}
			return l.T("mirror.msg.gpus", name, ch.After, ch.Restarted), nil
		}))

	mux.HandleFunc("POST /admin/mirror/state", p.mirrorAction(m,
		func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
			l := langOf(r)
			name := strings.TrimSpace(r.FormValue("model"))
			switch r.FormValue("action") {
			case "stop":
				if _, err := p.catalog.Stop(ctx, name, admin.Email); err != nil {
					return "", err
				}
				return l.T("mirror.msg.stopped", name), nil
			case "start":
				if _, err := p.catalog.Start(ctx, name, admin.Email); err != nil {
					return "", err
				}
				return l.T("mirror.msg.model_started", name), nil
			case "restart":
				s := p.settings()
				if s == nil {
					return "", errors.New(l.T("mirror.err.no_settings"))
				}
				n, err := s.RestartModel(name)
				if err != nil {
					return "", err
				}
				return l.N("mirror.msg.restarting", int64(n), name), nil
			}
			return "", errors.New(l.T("mirror.err.action"))
		}))

	mux.HandleFunc("POST /admin/mirror/split", p.mirrorAction(m,
		func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
			l := langOf(r)
			name := strings.TrimSpace(r.FormValue("model"))
			counts := map[string]int{}
			for _, prof := range r.Form["profile"] {
				n, err := count(l, r.FormValue("replicas_"+prof), "mirror.split.replicas")
				if err != nil {
					return "", err
				}
				counts[prof] = n
			}
			var cpu *modelsvc.CPUProfile
			if v := strings.TrimSpace(r.FormValue("cpu_replicas")); v != "" && v != "0" {
				n, err := count(l, v, "mirror.split.replicas")
				if err != nil {
					return "", err
				}
				mem, err := count(l, r.FormValue("cpu_memory"), "serve.memory")
				if err != nil {
					return "", err
				}
				cores, err := optionalCount(l, r.FormValue("cpu_cores"), "serve.cpu_cores")
				if err != nil {
					return "", err
				}
				cpu = &modelsvc.CPUProfile{Name: "cpu", MemoryGB: mem, CPUCores: cores}
				counts["cpu"] = n
			}
			got, err := p.catalog.SetProfileReplicas(ctx, name, counts, cpu, admin.Email)
			if err != nil {
				return "", err
			}
			var parts []string
			for _, pr := range got.Profiles {
				parts = append(parts, l.T("mirror.split.on", pr.Replicas, pr.Name))
			}
			return l.T("mirror.msg.split", name, strings.Join(parts, ", ")), nil
		}))

	mux.HandleFunc("POST /admin/mirror/unserve", p.mirrorAction(m,
		func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
			l := langOf(r)
			name, s, err := need(r)
			if err != nil {
				return "", err
			}
			purge := r.FormValue("purge") == "yes"
			out, err := s.RemoveModel(ctx, name, purge, admin.Email)
			if err != nil {
				return "", err
			}
			if purge {
				msg := l.N("mirror.msg.purged", int64(len(out.Mirror)), name)
				if out.Note != "" {
					msg += " " + out.Note
				}
				return msg, nil
			}
			return l.T("mirror.msg.unserved", name), nil
		}))
}

// count is a whole number of at least zero from a form field, or why not.
func count(l *Lang, v, field string) (int, error) {
	n, err := strconv.Atoi(strings.TrimSpace(v))
	if err != nil || n < 0 {
		return 0, errors.New(l.T("mirror.err.count", l.T(field)))
	}
	return n, nil
}

// optionalCount is count, with an empty field zero: left as it is.
func optionalCount(l *Lang, v, field string) (int, error) {
	if strings.TrimSpace(v) == "" {
		return 0, nil
	}
	return count(l, v, field)
}

// profileUse is one of a model's profiles as its card shows it.
type profileUse struct {
	Name     string
	CPU      bool
	Class    string
	GPUs     int
	Replicas int
}

// describeProfiles fills in what the card of a served model shows of its
// profiles, and of a profile on CPUs it could have.
func (p *AccountPages) describeProfiles(u *modelUse, m catalog.Model, st mirror.Status) {
	u.ContextLength, u.MaxOutput, u.Stopped, u.Split = m.ContextLength, m.MaxOutput, m.Stopped != nil, m.Split()
	hasCPU, llama := false, false
	for _, pr := range m.Profiles {
		pu := profileUse{Name: pr.Name, CPU: pr.CPUOnly(), Class: pr.GPUClass, Replicas: pr.Replicas,
			GPUs: max(pr.GPUsPerNode, pr.TP*max(pr.PP, 1))}
		u.Profiles = append(u.Profiles, pu)
		hasCPU = hasCPU || pu.CPU
		llama = llama || pr.Engine == "llamacpp"
		if !pu.CPU && u.GPUProfile == "" && pr.GPU == nil {
			u.GPUProfile, u.GPUs = pr.Name, max(pu.GPUs, 1)
		}
	}
	if !llama || hasCPU || m.IsExternal() {
		return
	}
	u.CanCPU = true
	u.CPUMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
	hosts := cpuHosts(p.fleet, u.CPUMemoryGB)
	var cpus []int
	for _, h := range hosts {
		cpus = append(cpus, h.CPUs)
	}
	u.NewCPUCores = catalog.SuggestCores(cpus)
	weights := st.BytesTotal
	if weights <= 0 {
		weights = st.BytesDone
	}
	u.CPUSpeed, u.CPUSpeedNode = bestCPUSpeed(hosts, weights, u.NewCPUCores)
}