The leader adds the picture and 3D presets a node fits, unless told not to

A customer's portal had no Images or 3D pages: nobody had written a
definition for either, on a cluster whose V100 runs both. The leading
control plane now adds flux-schnell and triposr, on demand, once a node
has a card that fits one, the cluster installs ComfyUI, and the catalog
makes nothing of that kind. presets.auto (a live setting, on by default)
turns it off; a preset somebody removes is recorded as declined and not
added back until added again by hand. Each addition is in the audit log.

GET /api/v1/presets says how each preset stands and why one is not added,
POST /api/v1/presets/{name} adds one, PUT /api/v1/presets/auto switches it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-13 16:31:08 +02:00
1 parent 92b588452e
commit a1974c2b45
17 files changed
+626 -11

No files matched your search

+11 -5
View File
@@ -81,11 +81,15 @@ type API struct {
// revoke records a removed node, so its certificate is refused.
revoke func(ctx context.Context, nodeID string, at time.Time) error
models Models
capacity Capacity
usage Usage
quotas Quotas
ctl *control.Controller
models Models
// presets keeps what became of each preset, presetCfg reads the
// settings in force; nil without a store.
presets PresetStore
presetCfg func() PresetConfig
capacity Capacity
usage Usage
quotas Quotas
ctl *control.Controller
}
// New builds the API over a controller.
@@ -1230,6 +1234,8 @@ func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
// A preset removed is one the leader must not put back by itself.
a.presetDeclined(r.Context(), name, actorOf(r))
writeJSON(w, http.StatusOK, Removed{Removed: name})
}
+241 -6
View File
@@ -4,10 +4,17 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// PresetsView is the models that come with llmgw, and whether the leader adds
@@ -38,6 +45,10 @@ type PresetView struct {
// Declined is whether somebody removed it after it was added, so it is
// not added again by itself; adding it again clears that.
Declined bool `json:"declined"`
// Blocked is why the leader does not add it by itself — presets.auto
// off, no ComfyUI, no card that fits, declined, another model already
// making the same — empty when it would or has.
Blocked string `json:"blocked,omitempty"`
}
// PresetsAutoRequest turns the automatic adding of presets on or off.
@@ -49,19 +60,243 @@ type PresetsAutoRequest struct {
// kept in.
var errPresetsUnavailable = errors.New("this control plane has no store, so presets cannot be added here")
// Presets is the presets as they stand in this cluster.
func (a *API) Presets(ctx context.Context) (PresetsView, error) {
return PresetsView{}, errPresetsUnavailable
// PresetStore keeps what became of each preset, and the audit log the
// leader writes its own additions to.
type PresetStore interface {
PresetStates(ctx context.Context) (map[string]store.PresetState, error)
PresetAdded(ctx context.Context, name, by string, at time.Time) error
PresetDeclined(ctx context.Context, name, by string, at time.Time) error
WriteAudit(ctx context.Context, e store.AuditEvent) error
}
// AddPreset adds the preset of that name, and clears its decline.
// PresetConfig is what the presets read of the settings in force.
type PresetConfig struct {
// Auto is presets.auto.
Auto bool
// ComfyUI is engines.comfyui: the version the nodes install, empty for
// none.
ComfyUI string
}
// SetPresets enables the presets: where what became of them is kept, and
// the settings in force.
func (a *API) SetPresets(st PresetStore, cfg func() PresetConfig) {
a.presets, a.presetCfg = st, cfg
}
// Presets is the presets as they stand in this cluster.
func (a *API) Presets(ctx context.Context) (PresetsView, error) {
if a.presets == nil || a.presetCfg == nil {
return PresetsView{}, errPresetsUnavailable
}
states, err := a.presets.PresetStates(ctx)
if err != nil {
return PresetsView{}, err
}
cfg := a.presetCfg()
cat := a.ctl.Catalog()
kinds := a.gpuKinds()
out := PresetsView{Auto: cfg.Auto}
for _, p := range catalog.Presets() {
v := PresetView{Name: p.Model.Name, Makes: p.Makes, Description: p.Model.Description,
Card: presetCard(p.Model), FitsClasses: presetFits(p.Model, kinds)}
if p.Model.License != nil {
v.Licence = *p.Model.License
}
v.Fits = len(v.FitsClasses) > 0
_, v.InCatalog = cat.Resolve(p.Model.Name)
if st, ok := states[p.Model.Name]; ok {
v.AddedBy, v.AddedAt, v.Declined = st.AddedBy, st.AddedAt, st.Declined
}
v.Blocked = presetBlocked(p, v, cfg, cat)
out.Presets = append(out.Presets, v)
}
return out, nil
}
// AddPreset adds the preset of that name, and clears its decline. A cluster
// that installs no ComfyUI yet is set to (engines.comfyui), as `llmgw ctl
// model add` does; that change starts the control planes again.
func (a *API) AddPreset(ctx context.Context, name, by string) error {
return errPresetsUnavailable
if a.presets == nil || a.presetCfg == nil || a.models == nil {
return errPresetsUnavailable
}
p, ok := catalog.PresetNamed(name)
if !ok {
return fmt.Errorf("%w: %q (there are %s)", errNoSuchPreset, name, strings.Join(catalog.PresetNames(), ", "))
}
if _, in := a.ctl.Catalog().Resolve(name); in {
return fmt.Errorf("%s is already in the catalog", name)
}
if a.presetCfg().ComfyUI == "" {
if err := a.setComfyUI(ctx, by); err != nil {
return err
}
}
if err := a.models.Add(ctx, p.Model, by); err != nil {
return err
}
return a.presets.PresetAdded(ctx, name, by, time.Now())
}
// SetAutoPresets turns presets.auto on or off.
func (a *API) SetAutoPresets(ctx context.Context, on bool, by string) error {
return errPresetsUnavailable
if a.presets == nil {
return errPresetsUnavailable
}
note := "presets added by themselves: " + map[bool]string{true: "on", false: "off"}[on]
return a.editSettings(ctx, []string{"presets", "auto"}, strconv.FormatBool(on), by, note)
}
// PresetsPass adds, as "auto", each preset a node fits whose kind of model
// the catalog has none of: run by the leader, now and then. It adds nothing
// with presets.auto off, where the cluster installs no ComfyUI, or a preset
// somebody declined. It says what it added.
func (a *API) PresetsPass(ctx context.Context) ([]string, error) {
v, err := a.Presets(ctx)
if err != nil || !v.Auto {
return nil, err
}
var added []string
for _, pv := range v.Presets {
if pv.Blocked != "" || pv.InCatalog {
continue
}
p, _ := catalog.PresetNamed(pv.Name)
if err := a.models.Add(ctx, p.Model, store.PresetAdder); err != nil {
return added, fmt.Errorf("adding %s: %w", pv.Name, err)
}
now := time.Now()
if err := a.presets.PresetAdded(ctx, pv.Name, store.PresetAdder, now); err != nil {
return added, err
}
_ = a.presets.WriteAudit(ctx, store.AuditEvent{At: now, Actor: store.PresetAdder, Source: "presets",
Action: "add preset", Target: pv.Name, Status: http.StatusOK,
Detail: fmt.Sprintf("%s, on demand: a node has %s (%s) and the catalog made no %s",
pv.Name, strings.Join(pv.FitsClasses, ", "), pv.Card, pv.Makes)})
added = append(added, pv.Name)
}
return added, nil
}
// presetDeclined records a preset's name removed from the catalog, so the
// leader does not put it back.
func (a *API) presetDeclined(ctx context.Context, name, by string) {
if a.presets == nil {
return
}
if _, ok := catalog.PresetNamed(name); ok {
_ = a.presets.PresetDeclined(ctx, name, by, time.Now())
}
}
// presetBlocked is why the leader would not add a preset by itself, empty
// when it would (or has).
func presetBlocked(p catalog.Preset, v PresetView, cfg PresetConfig, cat *catalog.Catalog) string {
switch {
case v.InCatalog:
return ""
case v.Declined:
return "removed from the catalog after it was added; add it again to undo"
case !cfg.Auto:
return "presets.auto is off"
case cfg.ComfyUI == "":
return "the cluster installs no ComfyUI (engines.comfyui); adding it by hand sets that"
case !v.Fits:
return "no node has a card it needs (" + v.Card + ")"
}
for _, m := range cat.Models() {
if p.Makes == catalog.OutputPictures && m.MakesImages() || p.Makes == catalog.OutputMeshes && m.MakesMeshes() {
return "the catalog has " + m.Name + ", which makes " + p.Makes + " already"
}
}
return ""
}
// presetCard is what a preset needs of a card, as people read it.
func presetCard(m catalog.Model) string {
var needs []string
for _, pr := range m.Profiles {
switch {
case pr.GPU != nil:
needs = append(needs, pr.GPU.String())
case pr.GPUClass != "":
needs = append(needs, pr.GPUClass)
}
}
return strings.Join(needs, " or ")
}
// presetFits are the classes of card in the cluster one of the preset's
// profiles can run on.
func presetFits(m catalog.Model, kinds []catalog.GPUKind) []string {
var out []string
for _, k := range kinds {
for _, pr := range m.Profiles {
if pr.GPU != nil && pr.GPU.Fits(k) || pr.GPU == nil && pr.GPUClass == k.Class {
out = append(out, k.Class)
break
}
}
}
return out
}
// gpuKinds are the kinds of card the cluster's nodes have.
func (a *API) gpuKinds() []catalog.GPUKind {
var gpus []discovery.GPU
for _, n := range a.ctl.Nodes() {
gpus = append(gpus, n.GPUs...)
}
return catalog.GPUKinds(gpus)
}
// setComfyUI has the nodes install the ComfyUI this release knows.
func (a *API) setComfyUI(ctx context.Context, by string) error {
if a.settings != nil {
cur, err := a.settings.Store.ClusterSettings(ctx)
if err != nil {
return err
}
if doc, err := settings.FromBytes("settings", []byte(cur.Body)); err == nil {
if n := doc.Node([]string{"engines", catalog.EngineComfyUI}); n != nil && n.Value == "" {
return errors.New("engines.comfyui is set to none, so the nodes install no ComfyUI: " +
"set a version (llmgw ctl config set engines.comfyui " + provision.ComfyUIVersion() + ")")
}
}
}
return a.editSettings(ctx, []string{"engines", catalog.EngineComfyUI}, provision.ComfyUIVersion(), by,
"ComfyUI for the presets")
}
// editSettings sets one key of the cluster's settings, as a new version.
func (a *API) editSettings(ctx context.Context, path []string, value, by, note string) error {
if a.settings == nil {
return errors.New("this control plane keeps no cluster settings")
}
cur, err := a.settings.Store.ClusterSettings(ctx)
if err != nil {
return err
}
doc, err := settings.FromBytes("settings", []byte(cur.Body))
if err != nil {
return err
}
if n := doc.Node(path); n != nil && n.Value == value {
return nil
}
if err := doc.SetYAML(path, value); err != nil {
return err
}
b, err := doc.Bytes()
if err != nil {
return err
}
if err := a.settings.Check(b); err != nil {
return err
}
_, err = a.settings.Store.PutClusterSettings(ctx, cur.Version, string(b), by, note)
return err
}
func (a *API) registerPresets(mux Mux) {
+138
View File
@@ -0,0 +1,138 @@
package adminapi
import (
"net/http"
"path/filepath"
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func v100Node(id string) inventory.Node {
return inventory.Node{ID: id, State: inventory.Ready, AgentVersion: "v1", GPUs: []discovery.GPU{{
Index: 0, Name: "Tesla V100-SXM2-32GB", MemoryTotalMB: 32768, ComputeCapability: "7.0"}}}
}
func presetsAPI(t *testing.T, models []catalog.Model, cfg PresetConfig, nodes ...inventory.Node) (*API, *fakeModels, *store.DB) {
t.Helper()
a, _ := setup(t, models, nodes...)
st, err := store.OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "p.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = st.Close() })
fm := &fakeModels{}
a.SetModels(fm)
a.SetPresets(st, func() PresetConfig { return cfg })
a.SetSettings(&Settings{Store: st, Check: func([]byte) error { return nil }})
return a, fm, st
}
func TestTheLeaderAddsThePresetsANodeFitsAndNoOtherWhere(t *testing.T) {
// A customer's portal had no pictures or 3D pages: nobody had written a
// definition for either, on a cluster with a V100 that runs both.
a, fm, st := presetsAPI(t, testModels(), PresetConfig{Auto: true, ComfyUI: "0.39.2"}, v100Node("p-g3"))
added, err := a.PresetsPass(t.Context())
if err != nil || !slices.Equal(added, []string{"flux-schnell", "triposr"}) {
t.Fatalf("added %v %v", added, err)
}
if !slices.Equal(fm.actors, []string{store.PresetAdder, store.PresetAdder}) {
t.Errorf("added by %v", fm.actors)
}
states, _ := st.PresetStates(t.Context())
if states["triposr"].AddedBy != store.PresetAdder || states["triposr"].AddedAt.IsZero() {
t.Errorf("state %+v", states["triposr"])
}
events, _ := st.AuditEvents(t.Context(), store.AuditFilter{Limit: 10})
if len(events) != 2 || events[0].Actor != store.PresetAdder || !strings.Contains(events[0].Detail, "v100-32g") {
t.Errorf("audit %+v", events)
}
}
func TestAPresetIsNotAddedByItselfWhereItShouldNotBe(t *testing.T) {
picture := catalog.Model{Name: "sdxl", Source: "/w/sdxl.safetensors", ComfyUI: &catalog.ComfyUI{Workflow: "flux-schnell"},
Profiles: []catalog.Profile{{Name: "p", Engine: catalog.EngineComfyUI, GPUClass: "v100-32g", TP: 1}}}
cases := []struct {
name string
models []catalog.Model
cfg PresetConfig
nodes []inventory.Node
want map[string]string // preset -> part of why it is not added
}{
{"auto off", testModels(), PresetConfig{Auto: false, ComfyUI: "0.39.2"}, []inventory.Node{v100Node("g")},
map[string]string{"flux-schnell": "presets.auto is off", "triposr": "presets.auto is off"}},
{"no ComfyUI", testModels(), PresetConfig{Auto: true}, []inventory.Node{v100Node("g")},
map[string]string{"triposr": "engines.comfyui"}},
{"no card", testModels(), PresetConfig{Auto: true, ComfyUI: "0.39.2"}, nil,
map[string]string{"triposr": "no node has a card"}},
{"a picture model already", append(testModels(), picture), PresetConfig{Auto: true, ComfyUI: "0.39.2"},
[]inventory.Node{v100Node("g")}, map[string]string{"flux-schnell": "sdxl"}},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
a, fm, _ := presetsAPI(t, c.models, c.cfg, c.nodes...)
v, err := a.Presets(t.Context())
if err != nil {
t.Fatal(err)
}
for _, p := range v.Presets {
if why, ok := c.want[p.Name]; ok && !strings.Contains(p.Blocked, why) {
t.Errorf("%s blocked %q, want it to say %q", p.Name, p.Blocked, why)
}
}
_, _ = a.PresetsPass(t.Context())
for _, m := range fm.added {
if _, blocked := c.want[m.Name]; blocked {
t.Errorf("%s was added", m.Name)
}
}
})
}
}
func TestAPresetRemovedStaysOutUntilAddedAgain(t *testing.T) {
a, fm, _ := presetsAPI(t, testModels(), PresetConfig{Auto: true, ComfyUI: "0.39.2"}, v100Node("g"))
if code := call(t, a, http.MethodDelete, "/api/v1/models/triposr", "", nil); code != http.StatusOK {
t.Fatalf("remove %d", code)
}
var v PresetsView
call(t, a, http.MethodGet, "/api/v1/presets", "", &v)
i := slices.IndexFunc(v.Presets, func(p PresetView) bool { return p.Name == "triposr" })
if i < 0 || !v.Presets[i].Declined || !strings.Contains(v.Presets[i].Blocked, "removed") {
t.Fatalf("after removal %+v", v.Presets)
}
added, _ := a.PresetsPass(t.Context())
if slices.Contains(added, "triposr") {
t.Error("a removed preset was added back by itself")
}
if code := call(t, a, http.MethodPost, "/api/v1/presets/triposr", "", nil); code != http.StatusOK {
t.Fatalf("add again %d", code)
}
if fm.added[len(fm.added)-1].Name != "triposr" {
t.Error("adding it again did not add it")
}
call(t, a, http.MethodGet, "/api/v1/presets", "", &v)
i = slices.IndexFunc(v.Presets, func(p PresetView) bool { return p.Name == "triposr" })
if v.Presets[i].Declined {
t.Error("adding it again left it declined")
}
if code := call(t, a, http.MethodPost, "/api/v1/presets/nope", "", nil); code != http.StatusNotFound {
t.Errorf("unknown preset %d", code)
}
}
func TestPresetsAutoIsTheClusterSetting(t *testing.T) {
a, _, st := presetsAPI(t, testModels(), PresetConfig{Auto: true, ComfyUI: "0.39.2"})
if code := call(t, a, http.MethodPut, "/api/v1/presets/auto", `{"on":false}`, nil); code != http.StatusOK {
t.Fatalf("put %d", code)
}
cur, _ := st.ClusterSettings(t.Context())
if !strings.Contains(cur.Body, "auto: false") {
t.Errorf("settings %q", cur.Body)
}
}