llmgw/internal/control/modelsvc/modelsvc.go

// Package modelsvc owns the catalog that is in force: the models declared in
// configuration, plus the ones added while the cluster is running.
//
// Two sources rather than one, deliberately. Configuration stays declarative —
// a model in the YAML is in the catalog on every start, whatever the database
// says — because that is how a team keeps its production set under review and
// in version control. The database holds what someone added afterwards, which
// is the case this exists for: trying a model should not mean a config change,
// a restart and an outage.
//
// Configuration wins every conflict, and the surprising direction is the
// dangerous one: a request for a reviewed production model must never be
// answered by something added from a laptop. A stored model that collides with
// a declared name is refused when it is added and skipped if it somehow
// survives into the database.
package modelsvc

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"sync"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// Store is the persistence this needs.
type Store interface {
	PutModel(ctx context.Context, m store.StoredModel) error
	DeleteModel(ctx context.Context, name string) error
	Models(ctx context.Context) ([]store.StoredModel, error)
}

// Applier receives the rebuilt catalog. The controller implements it.
type Applier interface {
	ReplaceCatalog(ctx context.Context, c *catalog.Catalog)
}

// Service adds and removes models without a restart.
type Service struct {
	declared []catalog.Model
	store    Store
	apply    Applier
	log      *slog.Logger

	// check refuses a model that cannot run before it is added; see
	// SetCheck.
	check func(context.Context, catalog.Model) error

	// mu serialises rebuilds. They are rare — a person adding a model — and two
	// at once could apply catalogs in either order, leaving the one in force
	// disagreeing with the database that outlives it.
	mu sync.Mutex
}

// New builds the service around the models configuration declared.
func New(declared []catalog.Model, st Store, apply Applier, log *slog.Logger) *Service {
	if log == nil {
		log = slog.Default()
	}
	return &Service{declared: declared, store: st, apply: apply, log: log}
}

// Declared reports whether a name comes from configuration.
func (s *Service) Declared(name string) bool {
	for _, d := range s.declared {
		if d.Name == name {
			return true
		}
	}
	return false
}

// Sources maps each weights source in the catalog to the models that name it,
// so a page listing files can say which of them are in use and how they run.
func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, error) {
	c, err := s.Load(ctx)
	if err != nil {
		return nil, err
	}
	out := map[string][]catalog.Model{}
	for _, m := range c.Models() {
		if m.Source != "" {
			out[m.Source] = append(out[m.Source], m)
		}
	}
	return out, nil
}

// Add puts a model in the catalog and keeps it across restarts.
//
// Validated by building the whole catalog before anything is written. A model
// that collides with an existing name, names a fallback that does not exist, or
// is malformed in any of the ways catalog.New refuses is rejected here rather
// than persisted and then skipped on every future start — which would be a
// model that looks added and never appears.
func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.Declared(m.Name) {
		return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
	}
	// A model with no GPU is all host memory, and one that declares none
	// fits everywhere as far as placement can tell — however many of it
	// there already are. Required of what is added now; models stored before
	// this still load, and say what they are missing.
	for _, p := range m.Profiles {
		if p.CPUOnly() && p.MinMemoryGB <= 0 {
			return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+
				"placement could not tell when a node is full", m.Name, p.Name)
		}
	}
	if s.check != nil {
		if err := s.check(ctx, m); err != nil {
			return err
		}
	}

	stored, err := s.storedModels(ctx)
	if err != nil {
		return err
	}
	next := append([]catalog.Model{}, s.declared...)
	for _, e := range stored {
		if e.Name == m.Name {
			continue // this call replaces it
		}
		next = append(next, e)
	}
	next = append(next, m)

	built, err := catalog.New(next)
	if err != nil {
		return fmt.Errorf("model %q would make the catalog invalid: %w", m.Name, err)
	}

	def, err := json.Marshal(m)
	if err != nil {
		return err
	}
	if err := s.store.PutModel(ctx, store.StoredModel{
		Name: m.Name, Definition: def, CreatedAt: time.Now(), CreatedBy: by,
	}); err != nil {
		return err
	}

	s.apply.ReplaceCatalog(ctx, built)
	s.log.Info("model added", "model", m.Name, "by", by)
	return nil
}

// Remove takes a model out of the catalog.
//
// A model declared in configuration cannot be removed this way: it would come
// back on the next restart, so the cluster would disagree with its own config
// file until then. Refusing and saying where to edit is the smaller surprise.
func (s *Service) Remove(ctx context.Context, name string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.Declared(name) {
		return fmt.Errorf("model %q is declared in configuration; remove it there, or it comes back on the next restart", name)
	}

	stored, err := s.storedModels(ctx)
	if err != nil {
		return err
	}
	next := append([]catalog.Model{}, s.declared...)
	found := false
	for _, e := range stored {
		if e.Name == name {
			found = true
			continue
		}
		next = append(next, e)
	}
	if !found {
		return fmt.Errorf("model %q was not added at runtime", name)
	}

	built, err := catalog.New(next)
	if err != nil {
		// Removing a model can invalidate a catalog: another may name it as a
		// fallback. Refusing beats leaving a fallback pointing at nothing,
		// which is discovered during the outage it was meant to cover.
		return fmt.Errorf("removing %q would make the catalog invalid: %w", name, err)
	}
	if err := s.store.DeleteModel(ctx, name); err != nil {
		return err
	}

	s.apply.ReplaceCatalog(ctx, built)
	s.log.Info("model removed", "model", name)
	return nil
}

// Scale changes how many replicas a model added at runtime runs.
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
	final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
		m.Scaling = sc
		return nil
	})
	if err == nil {
		s.log.Info("model scaled", "model", name,
			"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
	}
	return final, err
}

// Resources sets what a model's profile reserves of a node: host memory, and
// the CPU cores a replica without a GPU computes on. An empty profile name
// sets every profile.
func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) {
	if memoryGB < 0 || cpuCores < 0 {
		return catalog.Model{}, errors.New("memory and cores cannot be negative")
	}
	final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error {
		found := false
		for i := range m.Profiles {
			if profile != "" && m.Profiles[i].Name != profile {
				continue
			}
			found = true
			m.Profiles[i].MinMemoryGB = memoryGB
			m.Profiles[i].CPUCores = cpuCores
		}
		if !found {
			return fmt.Errorf("model %q has no profile %q", name, profile)
		}
		return nil
	})
	if err == nil {
		s.log.Info("model resources set", "model", name, "profile", profile,
			"memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by)
	}
	return final, err
}

// SetClass sets a model's capacity class: critical, standard or
// opportunistic, or empty to go back to what its scaling implies.
func (s *Service) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) {
	final, err := s.update(ctx, name, by, "change its class", func(m *catalog.Model) error {
		m.Class = class
		return nil
	})
	if err == nil {
		s.log.Info("model class set", "model", name, "class", final.EffectiveClass(), "by", by)
	}
	return final, err
}

// SetOwner sets the part of the tenancy tree a model's capacity counts
// against, org:ID, team:ID or project:ID; empty makes it shared.
func (s *Service) SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) {
	final, err := s.update(ctx, name, by, "change its owner", func(m *catalog.Model) error {
		m.Owner = owner
		return nil
	})
	if err == nil {
		s.log.Info("model owner set", "model", name, "owner", owner, "by", by)
	}
	return final, err
}

// update changes one model added at runtime and keeps the change.
//
// Built before anything is written, like Add: catalog.New is what applies the
// defaults and refuses what does not validate, so the model that comes back
// is the one that will run rather than the one that was asked for.
func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.Declared(name) {
		return catalog.Model{}, fmt.Errorf(
			"model %q is declared in configuration; %s there, or it reverts on the next restart", name, what)
	}
	stored, err := s.storedModels(ctx)
	if err != nil {
		return catalog.Model{}, err
	}
	next := append([]catalog.Model{}, s.declared...)
	found := false
	for _, e := range stored {
		if e.Name == name {
			if err := change(&e); err != nil {
				return catalog.Model{}, err
			}
			found = true
		}
		next = append(next, e)
	}
	if !found {
		return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
	}
	built, err := catalog.New(next)
	if err != nil {
		return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err)
	}
	final, ok := built.Resolve(name)
	if !ok {
		return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
	}
	def, err := json.Marshal(final)
	if err != nil {
		return catalog.Model{}, err
	}
	if err := s.store.PutModel(ctx, store.StoredModel{
		Name: name, Definition: def, CreatedAt: time.Now(), CreatedBy: by,
	}); err != nil {
		return catalog.Model{}, err
	}
	s.apply.ReplaceCatalog(ctx, built)
	return final, nil
}

// Load builds the catalog at startup: configuration plus what was stored.
//
// A stored model that no longer validates — its engine left the build, a
// field changed meaning — is reported and skipped. Refusing to start over one
// bad row would let a model somebody added from a laptop take down the cluster,
// and the models in configuration are precisely the ones that must survive it.
func (s *Service) Load(ctx context.Context) (*catalog.Catalog, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	stored, err := s.storedModels(ctx)
	if err != nil {
		return nil, err
	}

	next := append([]catalog.Model{}, s.declared...)
	for _, m := range stored {
		// Checked one at a time and against what is already accepted, so one
		// bad model cannot take a good one with it.
		candidate := append(append([]catalog.Model{}, next...), m)
		if _, err := catalog.New(candidate); err != nil {
			s.log.Warn("a stored model was skipped; it does not make a valid catalog",
				"model", m.Name, "err", err)
			continue
		}
		next = append(next, m)
	}
	return catalog.New(next)
}

// storedModels reads and decodes the runtime models, skipping any that no
// longer parse rather than failing every operation around them.
func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
	if s.store == nil {
		return nil, nil
	}
	rows, err := s.store.Models(ctx)
	if err != nil {
		return nil, err
	}
	out := make([]catalog.Model, 0, len(rows))
	for _, r := range rows {
		var m catalog.Model
		if err := json.Unmarshal(r.Definition, &m); err != nil {
			s.log.Warn("a stored model could not be decoded and was skipped",
				"model", r.Name, "err", err)
			continue
		}
		out = append(out, m)
	}
	return out, nil
}

// SetCheck runs before a model is added, and refuses it by returning an error.
// Used to catch weights the engine cannot load, which otherwise are placed and
// then fail on every node they are tried on.
func (s *Service) SetCheck(check func(context.Context, catalog.Model) error) { s.check = check }