Ilker ManapandClaude Opus 5.5
314cc5e6a3
Found on a fresh HA install with a judge machine: the leader saw the policy's judge missing while init was still fetching it, served one itself, and its entry replaced init's, so a second judge ran off the judge machine. The keeper now waits ten minutes after it starts, and a serve job it starts gives way to a model added meanwhile (AddNew). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
580 lines
19 KiB
Go
580 lines
19 KiB
Go
// Package modelsvc owns the catalog that is in force: the models declared in
|
|
// configuration, plus the ones added while the cluster is running.
|
|
//
|
|
// Two sources rather than one, deliberately. Configuration stays declarative —
|
|
// a model in the YAML is in the catalog on every start, whatever the database
|
|
// says — because that is how a team keeps its production set under review and
|
|
// in version control. The database holds what someone added afterwards, which
|
|
// is the case this exists for: trying a model should not mean a config change,
|
|
// a restart and an outage.
|
|
//
|
|
// Configuration wins every conflict, and the surprising direction is the
|
|
// dangerous one: a request for a reviewed production model must never be
|
|
// answered by something added from a laptop. A stored model that collides with
|
|
// a declared name is refused when it is added and skipped if it somehow
|
|
// survives into the database.
|
|
package modelsvc
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Store is the persistence this needs.
|
|
type Store interface {
|
|
PutModel(ctx context.Context, m store.StoredModel) error
|
|
DeleteModel(ctx context.Context, name string) error
|
|
Models(ctx context.Context) ([]store.StoredModel, error)
|
|
}
|
|
|
|
// Applier receives the rebuilt catalog. The controller implements it.
|
|
type Applier interface {
|
|
ReplaceCatalog(ctx context.Context, c *catalog.Catalog)
|
|
}
|
|
|
|
// Service adds and removes models without a restart.
|
|
type Service struct {
|
|
declared []catalog.Model
|
|
store Store
|
|
apply Applier
|
|
log *slog.Logger
|
|
|
|
// check refuses a model that cannot run before it is added; see
|
|
// SetCheck.
|
|
check func(context.Context, catalog.Model) error
|
|
|
|
// federated are the models federated clusters offer; see SetFederated.
|
|
federated func() []catalog.Model
|
|
|
|
// mu serialises rebuilds. They are rare — a person adding a model — and two
|
|
// at once could apply catalogs in either order, leaving the one in force
|
|
// disagreeing with the database that outlives it.
|
|
mu sync.Mutex
|
|
}
|
|
|
|
// New builds the service around the models configuration declared.
|
|
func New(declared []catalog.Model, st Store, apply Applier, log *slog.Logger) *Service {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Service{declared: declared, store: st, apply: apply, log: log}
|
|
}
|
|
|
|
// Declared reports whether a name comes from configuration.
|
|
func (s *Service) Declared(name string) bool {
|
|
for _, d := range s.declared {
|
|
if d.Name == name {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Sources maps each weights source in the catalog to the models that name it,
|
|
// so a page listing files can say which of them are in use and how they run.
|
|
func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, error) {
|
|
c, err := s.Load(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := map[string][]catalog.Model{}
|
|
for _, m := range c.Models() {
|
|
for _, src := range m.WeightSources() {
|
|
out[src] = append(out[src], m)
|
|
}
|
|
for _, ad := range m.Adapters {
|
|
out[ad.Source] = append(out[ad.Source], m)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Add puts a model in the catalog and keeps it across restarts.
|
|
//
|
|
// Validated by building the whole catalog before anything is written. A model
|
|
// that collides with an existing name, names a fallback that does not exist, or
|
|
// is malformed in any of the ways catalog.New refuses is rejected here rather
|
|
// than persisted and then skipped on every future start — which would be a
|
|
// model that looks added and never appears.
|
|
func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
|
return s.add(ctx, m, by, false)
|
|
}
|
|
|
|
// AddNew is Add for a model nobody has added meanwhile: one that is in the
|
|
// catalog by now is left as it is, and catalog.ErrExists said. The leader serving a
|
|
// judge a policy names gives way so to the judge init added while it fetched.
|
|
func (s *Service) AddNew(ctx context.Context, m catalog.Model, by string) error {
|
|
return s.add(ctx, m, by, true)
|
|
}
|
|
|
|
func (s *Service) add(ctx context.Context, m catalog.Model, by string, onlyNew bool) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
if s.Declared(m.Name) {
|
|
return fmt.Errorf("model %q is declared in the cluster's settings; edit it there (llmgw ctl config edit) rather than adding a second definition", m.Name)
|
|
}
|
|
// A model with no GPU is all host memory, and one that declares none
|
|
// fits everywhere as far as placement can tell — however many of it
|
|
// there already are. Required of what is added now; models stored before
|
|
// this still load, and say what they are missing.
|
|
for _, p := range m.Profiles {
|
|
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
|
return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+
|
|
"placement could not tell when a node is full", m.Name, p.Name)
|
|
}
|
|
}
|
|
if s.check != nil {
|
|
if err := s.check(ctx, m); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
stored, err := s.storedModels(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
next := s.base(stored)
|
|
for _, e := range stored {
|
|
if e.Name == m.Name {
|
|
if onlyNew {
|
|
return fmt.Errorf("model %q: %w", m.Name, catalog.ErrExists)
|
|
}
|
|
continue // this call replaces it
|
|
}
|
|
next = append(next, e)
|
|
}
|
|
next = append(next, m)
|
|
|
|
built, err := catalog.New(next)
|
|
if err != nil {
|
|
return fmt.Errorf("model %q would make the catalog invalid: %w", m.Name, err)
|
|
}
|
|
|
|
def, err := json.Marshal(m)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.store.PutModel(ctx, store.StoredModel{
|
|
Name: m.Name, Definition: def, CreatedAt: time.Now(), CreatedBy: by,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
s.apply.ReplaceCatalog(ctx, built)
|
|
s.log.Info("model added", "model", m.Name, "by", by)
|
|
return nil
|
|
}
|
|
|
|
// Remove takes a model out of the catalog.
|
|
//
|
|
// A model declared in configuration cannot be removed this way: it would come
|
|
// back on the next restart, so the cluster would disagree with its own config
|
|
// file until then. Refusing and saying where to edit is the smaller surprise.
|
|
func (s *Service) Remove(ctx context.Context, name string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
if s.Declared(name) {
|
|
return fmt.Errorf("model %q is declared in the cluster's settings, under models; remove it there: llmgw ctl config edit", name)
|
|
}
|
|
|
|
stored, err := s.storedModels(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
next := s.base(stored)
|
|
found := false
|
|
for _, e := range stored {
|
|
if e.Name == name {
|
|
found = true
|
|
continue
|
|
}
|
|
next = append(next, e)
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("model %q was not added at runtime", name)
|
|
}
|
|
|
|
built, err := catalog.New(next)
|
|
if err != nil {
|
|
// Removing a model can invalidate a catalog: another may name it as a
|
|
// fallback. Refusing beats leaving a fallback pointing at nothing,
|
|
// which is discovered during the outage it was meant to cover.
|
|
return fmt.Errorf("removing %q would make the catalog invalid: %w", name, err)
|
|
}
|
|
if err := s.store.DeleteModel(ctx, name); err != nil {
|
|
return err
|
|
}
|
|
|
|
s.apply.ReplaceCatalog(ctx, built)
|
|
s.log.Info("model removed", "model", name)
|
|
return nil
|
|
}
|
|
|
|
// Scale changes how many replicas a model added at runtime runs.
|
|
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
|
|
m.Scaling = sc
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
s.log.Info("model scaled", "model", name,
|
|
"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// Resources sets what a model's profile reserves of a node: host memory, and
|
|
// the CPU cores a replica without a GPU computes on. An empty profile name
|
|
// sets every profile.
|
|
func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) {
|
|
if memoryGB < 0 || cpuCores < 0 {
|
|
return catalog.Model{}, errors.New("memory and cores cannot be negative")
|
|
}
|
|
final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error {
|
|
found := false
|
|
for i := range m.Profiles {
|
|
if profile != "" && m.Profiles[i].Name != profile {
|
|
continue
|
|
}
|
|
found = true
|
|
m.Profiles[i].MinMemoryGB = memoryGB
|
|
m.Profiles[i].CPUCores = cpuCores
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("model %q has no profile %q", name, profile)
|
|
}
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
s.log.Info("model resources set", "model", name, "profile", profile,
|
|
"memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetProfiles replaces a model's profiles, as a model added with them
|
|
// would have: for a model whose engines are to run otherwise, its args among
|
|
// it. Running replicas keep what they were started with until restarted.
|
|
func (s *Service) SetProfiles(ctx context.Context, name string, profiles []catalog.Profile, by string) (catalog.Model, error) {
|
|
if len(profiles) == 0 {
|
|
return catalog.Model{}, errors.New("a model needs a profile")
|
|
}
|
|
return s.update(ctx, name, by, "replace its profiles", func(m *catalog.Model) error {
|
|
m.Profiles = slices.Clone(profiles)
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// SetClass sets a model's capacity class: critical, standard or
|
|
// opportunistic, or empty to go back to what its scaling implies.
|
|
func (s *Service) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "change its class", func(m *catalog.Model) error {
|
|
m.Class = class
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
s.log.Info("model class set", "model", name, "class", final.EffectiveClass(), "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetOwner sets the part of the tenancy tree a model's capacity counts
|
|
// against, org:ID, team:ID or project:ID; empty makes it shared.
|
|
func (s *Service) SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "change its owner", func(m *catalog.Model) error {
|
|
m.Owner = owner
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
s.log.Info("model owner set", "model", name, "owner", owner, "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetAdapters replaces the LoRA adapters a model serves. Replicas load the
|
|
// new set when they next start.
|
|
func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "change its adapters", func(m *catalog.Model) error {
|
|
m.Adapters = adapters
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
names := make([]string, 0, len(adapters))
|
|
for _, a := range adapters {
|
|
names = append(names, a.Name)
|
|
}
|
|
s.log.Info("model adapters set", "model", name, "adapters", names, "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetSource moves a model added at runtime to other weights, recording where
|
|
// they came from. Running replicas keep what they loaded until they are
|
|
// replaced; the caller restarts them.
|
|
func (s *Service) SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "change its source", func(m *catalog.Model) error {
|
|
m.Source, m.Origin = source, origin
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
rev := ""
|
|
if origin != nil {
|
|
rev = origin.Revision
|
|
}
|
|
s.log.Info("model source set", "model", name, "source", source, "revision", rev, "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetCanary gives a model added at runtime a canary, changes its share, or
|
|
// with nil takes it away. The candidate must already be in the catalog.
|
|
func (s *Service) SetCanary(ctx context.Context, name string, c *catalog.Canary, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "set its canary", func(m *catalog.Model) error {
|
|
m.Canary = c
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
if c == nil {
|
|
s.log.Info("model canary removed", "model", name, "by", by)
|
|
} else {
|
|
s.log.Info("model canary set", "model", name, "canary", c.Model, "percent", c.Percent, "by", by)
|
|
}
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetEval sets the quality suite a model's new weights must pass before an
|
|
// update or a promotion moves it to them; nil takes it away.
|
|
func (s *Service) SetEval(ctx context.Context, name string, g *catalog.EvalGate, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "set its eval gate", func(m *catalog.Model) error {
|
|
m.Eval = g
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
if g == nil {
|
|
s.log.Info("model eval gate removed", "model", name, "by", by)
|
|
} else {
|
|
s.log.Info("model eval gate set", "model", name, "suite", g.Suite, "min_score", g.MinScore, "by", by)
|
|
}
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetOrigin records where a model's weights came from without changing them:
|
|
// for a model added before origins were kept. A model from configuration is
|
|
// left alone, its origin worked out again when it is needed.
|
|
func (s *Service) SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error {
|
|
if s.Declared(name) {
|
|
return nil
|
|
}
|
|
_, err := s.update(ctx, name, "llmgw", "record its origin", func(m *catalog.Model) error {
|
|
m.Origin = origin
|
|
return nil
|
|
})
|
|
return err
|
|
}
|
|
|
|
// update changes one model added at runtime and keeps the change.
|
|
//
|
|
// Built before anything is written, like Add: catalog.New is what applies the
|
|
// defaults and refuses what does not validate, so the model that comes back
|
|
// is the one that will run rather than the one that was asked for.
|
|
func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
if s.Declared(name) {
|
|
return catalog.Model{}, fmt.Errorf(
|
|
"model %q is declared in the cluster's settings, under models; %s there: llmgw ctl config edit", name, what)
|
|
}
|
|
stored, err := s.storedModels(ctx)
|
|
if err != nil {
|
|
return catalog.Model{}, err
|
|
}
|
|
next := s.base(stored)
|
|
found := false
|
|
for _, e := range stored {
|
|
if e.Name == name {
|
|
if err := change(&e); err != nil {
|
|
return catalog.Model{}, err
|
|
}
|
|
found = true
|
|
}
|
|
next = append(next, e)
|
|
}
|
|
if !found {
|
|
return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
|
|
}
|
|
built, err := catalog.New(next)
|
|
if err != nil {
|
|
return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err)
|
|
}
|
|
final, ok := built.Resolve(name)
|
|
if !ok {
|
|
return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
|
|
}
|
|
def, err := json.Marshal(final)
|
|
if err != nil {
|
|
return catalog.Model{}, err
|
|
}
|
|
if err := s.store.PutModel(ctx, store.StoredModel{
|
|
Name: name, Definition: def, CreatedAt: time.Now(), CreatedBy: by,
|
|
}); err != nil {
|
|
return catalog.Model{}, err
|
|
}
|
|
s.apply.ReplaceCatalog(ctx, built)
|
|
return final, nil
|
|
}
|
|
|
|
// Load builds the catalog at startup: configuration plus what was stored.
|
|
//
|
|
// A stored model that no longer validates — its engine left the build, a
|
|
// field changed meaning — is reported and skipped. Refusing to start over one
|
|
// bad row would let a model somebody added from a laptop take down the cluster,
|
|
// and the models in configuration are precisely the ones that must survive it.
|
|
func (s *Service) Load(ctx context.Context) (*catalog.Catalog, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
return s.load(ctx)
|
|
}
|
|
|
|
// load is Load, with the lock held.
|
|
func (s *Service) load(ctx context.Context) (*catalog.Catalog, error) {
|
|
stored, err := s.storedModels(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// The federated models before the stored ones, so a stored model that
|
|
// falls back to a peer's is checked against a catalog that has it.
|
|
next := s.base(stored)
|
|
for _, m := range stored {
|
|
// Checked one at a time and against what is already accepted, so one
|
|
// bad model cannot take a good one with it.
|
|
candidate := append(append([]catalog.Model{}, next...), m)
|
|
if _, err := catalog.New(candidate); err != nil {
|
|
s.log.Warn("a stored model was skipped; it does not make a valid catalog",
|
|
"model", m.Name, "err", err)
|
|
continue
|
|
}
|
|
next = append(next, m)
|
|
}
|
|
return catalog.New(next)
|
|
}
|
|
|
|
// storedModels reads and decodes the runtime models, skipping any that no
|
|
// longer parse rather than failing every operation around them.
|
|
func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
|
|
if s.store == nil {
|
|
return nil, nil
|
|
}
|
|
rows, err := s.store.Models(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]catalog.Model, 0, len(rows))
|
|
for _, r := range rows {
|
|
var m catalog.Model
|
|
if err := json.Unmarshal(r.Definition, &m); err != nil {
|
|
s.log.Warn("a stored model could not be decoded and was skipped",
|
|
"model", r.Name, "err", err)
|
|
continue
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// SetFederated supplies the models federated clusters offer, which join the
|
|
// catalog after the declared ones and before those added at runtime.
|
|
//
|
|
// A function rather than a list, read at every rebuild, because the list
|
|
// changes whenever a peer's does; Refresh is how federation says it has.
|
|
func (s *Service) SetFederated(f func() []catalog.Model) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.federated = f
|
|
}
|
|
|
|
// Refresh rebuilds the catalog and puts it in force: after a peer's models
|
|
// changed. Built as at startup, so a stored model that no longer makes a
|
|
// valid catalog is skipped and said rather than taking the rest with it.
|
|
func (s *Service) Refresh(ctx context.Context) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
built, err := s.load(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.apply.ReplaceCatalog(ctx, built)
|
|
return nil
|
|
}
|
|
|
|
// base is the declared models, then the federated ones whose names nothing
|
|
// declared or added at runtime already uses.
|
|
//
|
|
// A peer's model is named PEER/MODEL, which a local model could also be
|
|
// called — Hugging Face ids look just like that. The local one wins, for the
|
|
// reason configuration wins over the database: a request for a model this
|
|
// cluster runs must not leave it because a peer started offering a name.
|
|
func (s *Service) base(stored []catalog.Model) []catalog.Model {
|
|
next := append([]catalog.Model{}, s.declared...)
|
|
if s.federated == nil {
|
|
return next
|
|
}
|
|
taken := map[string]bool{}
|
|
for _, list := range [][]catalog.Model{s.declared, stored} {
|
|
for _, m := range list {
|
|
taken[m.Name] = true
|
|
for _, a := range m.Aliases {
|
|
taken[a] = true
|
|
}
|
|
}
|
|
}
|
|
for _, f := range s.federated() {
|
|
if taken[f.Name] {
|
|
s.log.Warn("a federated model has the name of a model of this cluster's; not offered",
|
|
"model", f.Name)
|
|
continue
|
|
}
|
|
next = append(next, f)
|
|
}
|
|
return next
|
|
}
|
|
|
|
// SetFallbacks replaces the models a model added at runtime falls back to
|
|
// when it has no capacity: other models of the catalog, a federated peer's
|
|
// among them.
|
|
func (s *Service) SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error) {
|
|
final, err := s.update(ctx, name, by, "change its fallbacks", func(m *catalog.Model) error {
|
|
for _, f := range fallbacks {
|
|
if f == m.Name {
|
|
return fmt.Errorf("model %q cannot fall back to itself", name)
|
|
}
|
|
}
|
|
m.Fallbacks = fallbacks
|
|
return nil
|
|
})
|
|
if err == nil {
|
|
s.log.Info("model fallbacks set", "model", name, "fallbacks", fallbacks, "by", by)
|
|
}
|
|
return final, err
|
|
}
|
|
|
|
// SetCheck runs before a model is added, and refuses it by returning an error.
|
|
// Used to catch weights the engine cannot load, which otherwise are placed and
|
|
// then fail on every node they are tried on.
|
|
func (s *Service) SetCheck(check func(context.Context, catalog.Model) error) { s.check = check }
|