The cluster's settings live in the store, and every change takes effect

Everything in server.yaml that is not one machine's own is now a versioned
document in the store (cluster_settings), the same on every control plane.
The first start against a store imports the file's cluster keys, with the
baseline guard policy when there is none; after that those keys in a file
are ignored, with a warning.

Each control plane polls the document's version and its own server.yaml's
hash. A change is validated and applied: guard live (Gate.Replace), anything
else by a graceful restart that systemd brings back, standbys first and the
leader 30s later. A version that does not load on a machine is not applied;
at start the newest one that loads is used, so a bad edit cannot keep every
control plane down.

llmgw ctl config show/get/set/unset/edit/history/rollback, the admin API
under /api/v1/settings, and the console's Settings screen, which layers the
store's document over the machine's file, write it. Commands beside the
control plane that read cluster keys (init, node add, server add, backup,
upgrade) read the layered configuration; init --bundle writes offline to
the store once it holds the settings.

A standby that becomes the leader re-reads models, engine choices and
facility holds from the store.

Retires appendGuardIfMissing: the import does it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 15:25:23 +02:00
1 parent cbbfce235e
commit 6289b01ed6
57 files changed
+2579 -224

No files matched your search

+80 -4
View File
@@ -16,6 +16,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -80,10 +81,13 @@ func newServerCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
log := logger()
cfg, err := config.Load(configPath)
// server.yaml for this machine's own keys, the store's settings
// for the cluster's: the same on every control plane.
boot, err := loadClusterSettings(cmd.Context(), configPath, log)
if err != nil {
return err
}
cfg := boot.cfg
if listen != "" {
cfg.Listen = listen
}
@@ -166,7 +170,7 @@ func newServerCmd() *cobra.Command {
}
gate.SetJudge(rt.Judge, rt.GuardFormat)
rt.SetGuard(gate)
rt.SetGuardLibrary(cfg.Guard)
rt.SetGuardLibrary(gate.Library)
if err := rt.RegisterMetrics(gate); err != nil {
return fmt.Errorf("guard metrics: %w", err)
}
@@ -175,6 +179,10 @@ func newServerCmd() *cobra.Command {
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// Cancelled to start again on changed settings.
ctx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
var restartReason atomic.Pointer[string]
rt.Start(ctx)
defer rt.Stop()
@@ -341,12 +349,16 @@ func newServerCmd() *cobra.Command {
evals = newEvalService(cfg, rt, admin, log)
api.SetEval(evals)
if st := rt.Store(); st != nil && gate != nil {
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
api.SetGuard(&adminapi.GuardAdmin{Library: gate.Library, Store: st, Invalidate: gate.Invalidate})
}
if st := rt.Store(); st != nil {
// Removing a node revokes its certificate.
api.SetRevoker(st.RevokeNode)
api.SetAudit(st)
// The cluster's settings, each change checked against this
// machine's own keys before it is kept.
api.SetSettings(&adminapi.Settings{Store: st, LocalKeys: config.LocalKeys(),
Check: func(body []byte) error { return checkClusterSettings(configPath, body) }})
}
// Catalog changes need somewhere to persist, so this is wired
// only when there is a store. Without one, the endpoints say so
@@ -378,6 +390,17 @@ func newServerCmd() *cobra.Command {
} else {
admin.ReplaceCatalog(ctx, built)
}
// And again on becoming the leader: models added or
// removed through the leader before are in the store.
if l := admin.Leadership(); l != nil {
l.OnLead(func() {
if built, err := svc.Load(ctx); err != nil {
log.Error("could not read the stored models on becoming the leader", "err", err)
} else {
admin.ReplaceCatalog(ctx, built)
}
})
}
api.SetModels(svc)
catalogSvc = svc
// And the mirror page, so a model that has just been
@@ -627,6 +650,46 @@ func newServerCmd() *cobra.Command {
}
}
// Every change to the settings in force: what can change live
// does, the rest starts this process again, which systemd brings
// back on them.
watch := &settingsWatch{path: configPath, st: rt.Store(), cur: boot, log: log,
live: func(next config.Server, changed []string) bool {
if !onlyLive(changed) {
return false
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "guard") }) {
if gate == nil {
// Built at start, and only when there are policies.
return len(next.Guard.Policies) == 0
}
if err := gate.Replace(next.Guard); err != nil {
log.Error("the changed guard policies are not valid; the old ones stay in force", "err", err)
}
}
return true
},
restart: func(reason string) {
if os.Getenv("INVOCATION_ID") == "" {
log.Warn("the settings changed in a way this process applies only when it starts; "+
"it is not run by systemd, so start it again by hand", "why", reason)
return
}
// The leader last, so its standbys are already on the new
// settings when it hands over to one of them.
delay := time.Duration(0)
if admin != nil && admin.Leadership() != nil && admin.Leadership().IsLeader() && rt.Store() != nil {
if cps, err := rt.Store().ControlPlanes(ctx); err == nil && len(cps) > 1 {
delay = 30 * time.Second
}
}
log.Warn("starting again to apply the settings", "why", reason, "in", delay)
restartReason.Store(&reason)
time.AfterFunc(delay, cancelRun)
},
}
go watch.run(ctx)
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error { return serve(apiSrv, apiLn) })
g.Go(func() error { return serve(adminSrv, adminLn) })
@@ -658,7 +721,12 @@ func newServerCmd() *cobra.Command {
_ = adminSrv.Shutdown(sctx)
return apiSrv.Shutdown(sctx)
})
return g.Wait()
err = g.Wait()
if r := restartReason.Load(); r != nil {
// Not a failure, but said as one, so systemd starts it again.
return fmt.Errorf("stopped to start again on changed settings (%s)", *r)
}
return err
},
}
@@ -756,6 +824,14 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log))
// Domains held for maintenance and labels set on nodes.
ctl.Facility().UseStore(ctx, st)
// A standby read these when it started; the leader before it may
// have changed them since.
lead.OnLead(func() {
if ch := ctl.EngineChoices(); ch != nil {
ch.Reload(ctx)
}
ctl.Facility().Reload(ctx)
})
log.Info("competing to be the control plane that places work",
"owner", lead.Owner())
}