package web import ( "context" "errors" "net/http" "strconv" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // ModelSettings changes a served model beyond what the catalogue holds: its // context, its cards, its replicas started again, its removal. The cluster // view answers it where it can. type ModelSettings interface { SetModelContext(ctx context.Context, name string, req modelserve.ContextRequest, by string) (modelserve.ContextChange, error) SetModelGPUs(ctx context.Context, name string, req modelserve.GPUsRequest, by string) (modelserve.GPUsChange, error) RestartModel(name string) (int, error) RemoveModel(ctx context.Context, name string, purge bool, by string) (adminapi.PurgeView, error) } // settings is the cluster view's ModelSettings, nil where it has none. func (p *AccountPages) settings() ModelSettings { s, _ := p.fleet.(ModelSettings) return s } // registerModelSettings mounts what the card of a served model changes. func (p *AccountPages) registerModelSettings(mux *http.ServeMux, m MirrorAdmin) { need := func(r *http.Request) (string, ModelSettings, error) { name := strings.TrimSpace(r.FormValue("model")) if name == "" { return "", nil, errors.New(langOf(r).T("mirror.err.choose_model")) } s := p.settings() if s == nil { return "", nil, errors.New(langOf(r).T("mirror.err.no_settings")) } return name, s, nil } mux.HandleFunc("POST /admin/mirror/context", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name, s, err := need(r) if err != nil { return "", err } tokens, err := optionalCount(l, r.FormValue("tokens"), "serve.context_length") if err != nil { return "", err } maxOut, err := optionalCount(l, r.FormValue("max_output"), "serve.max_output") if err != nil { return "", err } ch, err := s.SetModelContext(ctx, name, modelserve.ContextRequest{Tokens: tokens, MaxOutput: maxOut}, admin.Email) if err != nil { return "", err } return l.T("mirror.msg.context", name, l.Int(int64(ch.After)), l.Int(int64(ch.MaxOutput)), ch.Restarted), nil })) mux.HandleFunc("POST /admin/mirror/gpus", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name, s, err := need(r) if err != nil { return "", err } n, err := strconv.Atoi(strings.TrimSpace(r.FormValue("gpus"))) if err != nil || n < 1 { return "", errors.New(l.T("mirror.err.gpus")) } ch, err := s.SetModelGPUs(ctx, name, modelserve.GPUsRequest{GPUs: n, Profile: r.FormValue("profile")}, admin.Email) if err != nil { return "", err } return l.T("mirror.msg.gpus", name, ch.After, ch.Restarted), nil })) mux.HandleFunc("POST /admin/mirror/state", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name := strings.TrimSpace(r.FormValue("model")) switch r.FormValue("action") { case "stop": if _, err := p.catalog.Stop(ctx, name, admin.Email); err != nil { return "", err } return l.T("mirror.msg.stopped", name), nil case "start": if _, err := p.catalog.Start(ctx, name, admin.Email); err != nil { return "", err } return l.T("mirror.msg.model_started", name), nil case "restart": s := p.settings() if s == nil { return "", errors.New(l.T("mirror.err.no_settings")) } n, err := s.RestartModel(name) if err != nil { return "", err } return l.N("mirror.msg.restarting", int64(n), name), nil } return "", errors.New(l.T("mirror.err.action")) })) mux.HandleFunc("POST /admin/mirror/split", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name := strings.TrimSpace(r.FormValue("model")) counts := map[string]int{} for _, prof := range r.Form["profile"] { n, err := count(l, r.FormValue("replicas_"+prof), "mirror.split.replicas") if err != nil { return "", err } counts[prof] = n } var cpu *modelsvc.CPUProfile if v := strings.TrimSpace(r.FormValue("cpu_replicas")); v != "" && v != "0" { n, err := count(l, v, "mirror.split.replicas") if err != nil { return "", err } mem, err := count(l, r.FormValue("cpu_memory"), "serve.memory") if err != nil { return "", err } cores, err := optionalCount(l, r.FormValue("cpu_cores"), "serve.cpu_cores") if err != nil { return "", err } cpu = &modelsvc.CPUProfile{Name: "cpu", MemoryGB: mem, CPUCores: cores} counts["cpu"] = n } got, err := p.catalog.SetProfileReplicas(ctx, name, counts, cpu, admin.Email) if err != nil { return "", err } var parts []string for _, pr := range got.Profiles { parts = append(parts, l.T("mirror.split.on", pr.Replicas, pr.Name)) } return l.T("mirror.msg.split", name, strings.Join(parts, ", ")), nil })) mux.HandleFunc("POST /admin/mirror/unserve", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { l := langOf(r) name, s, err := need(r) if err != nil { return "", err } purge := r.FormValue("purge") == "yes" out, err := s.RemoveModel(ctx, name, purge, admin.Email) if err != nil { return "", err } if purge { msg := l.N("mirror.msg.purged", int64(len(out.Mirror)), name) if out.Note != "" { msg += " " + out.Note } return msg, nil } return l.T("mirror.msg.unserved", name), nil })) } // count is a whole number of at least zero from a form field, or why not. func count(l *Lang, v, field string) (int, error) { n, err := strconv.Atoi(strings.TrimSpace(v)) if err != nil || n < 0 { return 0, errors.New(l.T("mirror.err.count", l.T(field))) } return n, nil } // optionalCount is count, with an empty field zero: left as it is. func optionalCount(l *Lang, v, field string) (int, error) { if strings.TrimSpace(v) == "" { return 0, nil } return count(l, v, field) } // profileUse is one of a model's profiles as its card shows it. type profileUse struct { Name string CPU bool Class string GPUs int Replicas int } // describeProfiles fills in what the card of a served model shows of its // profiles, and of a profile on CPUs it could have. func (p *AccountPages) describeProfiles(u *modelUse, m catalog.Model, st mirror.Status) { u.ContextLength, u.MaxOutput, u.Stopped, u.Split = m.ContextLength, m.MaxOutput, m.Stopped != nil, m.Split() hasCPU, llama := false, false for _, pr := range m.Profiles { pu := profileUse{Name: pr.Name, CPU: pr.CPUOnly(), Class: pr.GPUClass, Replicas: pr.Replicas, GPUs: max(pr.GPUsPerNode, pr.TP*max(pr.PP, 1))} u.Profiles = append(u.Profiles, pu) hasCPU = hasCPU || pu.CPU llama = llama || pr.Engine == "llamacpp" if !pu.CPU && u.GPUProfile == "" && pr.GPU == nil { u.GPUProfile, u.GPUs = pr.Name, max(pu.GPUs, 1) } } if !llama || hasCPU || m.IsExternal() { return } u.CanCPU = true u.CPUMemoryGB = memoryFor(st.BytesTotal, st.BytesDone) hosts := cpuHosts(p.fleet, u.CPUMemoryGB) var cpus []int for _, h := range hosts { cpus = append(cpus, h.CPUs) } u.NewCPUCores = catalog.SuggestCores(cpus) weights := st.BytesTotal if weights <= 0 { weights = st.BytesDone } u.CPUSpeed, u.CPUSpeedNode = bestCPUSpeed(hosts, weights, u.NewCPUCores) }