The cluster's settings live in the store, and every change takes effect

Everything in server.yaml that is not one machine's own is now a versioned
document in the store (cluster_settings), the same on every control plane.
The first start against a store imports the file's cluster keys, with the
baseline guard policy when there is none; after that those keys in a file
are ignored, with a warning.

Each control plane polls the document's version and its own server.yaml's
hash. A change is validated and applied: guard live (Gate.Replace), anything
else by a graceful restart that systemd brings back, standbys first and the
leader 30s later. A version that does not load on a machine is not applied;
at start the newest one that loads is used, so a bad edit cannot keep every
control plane down.

llmgw ctl config show/get/set/unset/edit/history/rollback, the admin API
under /api/v1/settings, and the console's Settings screen, which layers the
store's document over the machine's file, write it. Commands beside the
control plane that read cluster keys (init, node add, server add, backup,
upgrade) read the layered configuration; init --bundle writes offline to
the store once it holds the settings.

A standby that becomes the leader re-reads models, engine choices and
facility holds from the store.

Retires appendGuardIfMissing: the import does it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 15:25:23 +02:00
1 parent cbbfce235e
commit 6289b01ed6
57 files changed
+2579 -224

No files matched your search

+1 -1
View File
@@ -54,7 +54,7 @@ func newBackupCmd() *cobra.Command {
"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
"BackupTooOld fires when it is more than a day old.",
RunE: func(cmd *cobra.Command, _ []string) error {
cfg, err := config.Load(configPath)
cfg, err := loadEffectiveConfig(configPath)
if err != nil {
return err
}
+1 -1
View File
@@ -323,7 +323,7 @@ func loadServerConfig(path string) (config.Server, bool) {
if _, err := os.Stat(path); err != nil {
return config.Server{}, false
}
cfg, err := config.Load(path)
cfg, err := loadEffectiveConfig(path)
if err != nil {
return config.Server{}, false
}
+361
View File
@@ -0,0 +1,361 @@
package cli
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"log/slog"
"os"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// settingsPoll is how often a control plane asks whether the cluster's
// settings, or its own server.yaml, changed.
const settingsPoll = 5 * time.Second
// clusterSettings is what a control plane runs on, and where it came from:
// its server.yaml for its own keys, the store's settings for the rest.
type clusterSettings struct {
cfg config.Server
eff map[string]any // the settings, as written, cfg was made from
cluster map[string]any // the store's document, nil without a store
version int64
fileHash [32]byte
}
// loadClusterSettings reads the configuration a control plane starts on.
// Without a store it is the file. With one, the store's settings take the
// place of the file's cluster keys; a store with none yet is given the
// file's, so the first control plane to start against it brings them in.
func loadClusterSettings(ctx context.Context, path string, log *slog.Logger) (clusterSettings, error) {
var s clusterSettings
b, file, err := readSettingsFile(path)
if err != nil {
return s, err
}
s.fileHash = sha256.Sum256(b)
fileCfg, err := config.FromMap(file)
if err != nil {
return s, fileError(path, err)
}
if fileCfg.Auth.Store == "" {
s.cfg, s.eff = fileCfg, file
return s, nil
}
st, err := storepkg.Open(ctx, fileCfg.Auth.Store)
if err != nil {
return s, fmt.Errorf("the store, for the cluster's settings: %w", err)
}
defer st.Close()
doc, err := st.ClusterSettings(ctx)
if err != nil {
return s, fmt.Errorf("reading the cluster's settings: %w", err)
}
if doc.Version == 0 {
if doc, err = importSettings(ctx, st, path, file, log); err != nil {
return s, err
}
} else if ignored := config.Ignored(file); len(ignored) > 0 {
log.Warn("server.yaml sets keys the cluster's settings now hold; they are not read "+
"(llmgw ctl config shows what is in force)", "keys", strings.Join(ignored, ", "))
}
// The newest version that this machine can run on. One that cannot — a
// path only another control plane has — must not keep this one down,
// where nothing could be used to put it right.
versions := []storepkg.SettingsVersion{doc}
if older, err := st.ClusterSettingsHistory(ctx, 20); err == nil && len(older) > 1 {
versions = append(versions, older[1:]...)
}
var firstErr error
for _, v := range versions {
cluster, err := config.Parse([]byte(v.Body))
if err == nil {
eff := config.Layer(file, cluster)
var cfg config.Server
if cfg, err = config.FromMap(eff); err == nil {
if v.Version != doc.Version {
log.Error("the cluster's newest settings do not load on this machine; running on an older version",
"newest", doc.Version, "running", v.Version, "err", firstErr)
}
s.cfg, s.eff, s.cluster, s.version = cfg, eff, cluster, doc.Version
return s, nil
}
}
if firstErr == nil {
firstErr = err
}
}
return s, fmt.Errorf("the cluster's settings (version %d) do not load on this machine: %w", doc.Version, firstErr)
}
// importSettings records a server.yaml's cluster keys as the cluster's first
// settings, with the baseline guard policy when the file has none: a cluster
// installed before guardrails gets them on its first start after an upgrade.
func importSettings(ctx context.Context, st *storepkg.DB, path string, file map[string]any,
log *slog.Logger,
) (storepkg.SettingsVersion, error) {
_, cluster := config.Split(file)
note := "imported from " + path
if _, ok := cluster["guard"]; !ok {
baseline, err := config.Parse([]byte(guardSection(false)))
if err == nil {
cluster["guard"] = baseline["guard"]
note += ", with the baseline guard policy it did not have"
}
}
body, err := config.Marshal(cluster)
if err != nil {
return storepkg.SettingsVersion{}, err
}
host, _ := os.Hostname()
_, err = st.PutClusterSettings(ctx, 0, string(body), host, note)
switch {
case errors.Is(err, storepkg.ErrSettingsChanged):
// Another control plane brought its own in first: those are the
// cluster's.
case err != nil:
return storepkg.SettingsVersion{}, fmt.Errorf("recording the cluster's settings: %w", err)
default:
log.Info("the cluster's settings are now kept in the store; server.yaml keeps this machine's own",
"from", path)
}
return st.ClusterSettings(ctx)
}
func readSettingsFile(path string) ([]byte, map[string]any, error) {
if path == "" {
return nil, map[string]any{}, nil
}
b, err := os.ReadFile(path) //nolint:gosec // the configuration
if err != nil {
return nil, nil, fmt.Errorf("config file: %w", err)
}
m, err := config.Parse(b)
if err != nil {
return nil, nil, fmt.Errorf("parsing %s: %w", path, err)
}
return b, m, nil
}
func fileError(path string, err error) error {
if path == "" {
return err
}
return fmt.Errorf("%s: %w", path, err)
}
// settingsWatch follows the cluster's settings and this machine's
// server.yaml, and puts every change in force.
type settingsWatch struct {
path string
st *storepkg.DB // nil: the file only
cur clusterSettings
// live applies a change without a restart, reporting whether it could.
live func(cfg config.Server, changed []string) bool
// restart has the process start again on the new settings.
restart func(reason string)
log *slog.Logger
}
func (w *settingsWatch) run(ctx context.Context) {
t := time.NewTicker(settingsPoll)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
w.check(ctx)
}
}
}
// check reads what changed, if anything, and applies it.
func (w *settingsWatch) check(ctx context.Context) {
version := w.cur.version
if w.st != nil {
v, err := w.st.ClusterSettingsVersion(ctx)
if err != nil {
return // asked again in a few seconds
}
version = v
}
b, file, err := readSettingsFile(w.path)
if err != nil {
w.log.Error("server.yaml cannot be read; running on what it said before", "err", err)
return
}
hash := sha256.Sum256(b)
if version == w.cur.version && hash == w.cur.fileHash {
return
}
cluster := w.cur.cluster
eff := file
if w.st != nil {
if version != w.cur.version {
doc, err := w.st.ClusterSettings(ctx)
if err != nil {
return
}
if cluster, err = config.Parse([]byte(doc.Body)); err != nil {
w.log.Error("the cluster's settings do not parse; running on what was in force",
"version", doc.Version, "err", err)
w.cur.version, w.cur.fileHash = version, hash
return
}
}
eff = config.Layer(file, cluster)
}
cfg, err := config.FromMap(eff)
if err != nil {
w.log.Error("the changed settings do not load on this machine; running on what was in force",
"version", version, "err", err)
w.cur.version, w.cur.fileHash = version, hash
return
}
changed := config.Changed(w.cur.eff, eff)
w.cur = clusterSettings{cfg: cfg, eff: eff, cluster: cluster, version: version, fileHash: hash}
if len(changed) == 0 {
return
}
if w.live != nil && w.live(cfg, changed) {
w.log.Info("settings changed and in force", "version", version, "changed", strings.Join(changed, ", "))
return
}
w.restart(fmt.Sprintf("settings version %d changed %s", version, strings.Join(changed, ", ")))
}
// liveSettings are the keys a control plane puts in force without starting
// again: what it reads at every use, or not at all.
var liveSettings = []string{
"guard",
// Read by server init, node add and backups, not by the server.
"firewall_networks", "backup_recipients", "backup_copy_to",
}
// onlyLive is whether every changed key is one of liveSettings.
func onlyLive(changed []string) bool {
for _, c := range changed {
top, _, _ := strings.Cut(c, ".")
if !slices.Contains(liveSettings, top) {
return false
}
}
return true
}
// checkClusterSettings refuses a settings document that would not load,
// laid over the server.yaml at path, or that sets one machine's keys.
func checkClusterSettings(path string, body []byte) error {
cluster, err := config.Parse(body)
if err != nil {
return err
}
if err := config.CheckCluster(cluster); err != nil {
return err
}
_, file, err := readSettingsFile(path)
if err != nil {
return err
}
_, err = config.FromMap(config.Layer(file, cluster))
return err
}
// loadEffectiveConfig is what the control plane on this machine runs on:
// its server.yaml with the cluster's settings from the store laid over it,
// for the commands beside it that read cluster keys — init, node add,
// backups. The file alone when the store cannot be reached or holds none.
func loadEffectiveConfig(path string) (config.Server, error) {
_, file, err := readSettingsFile(path)
if err != nil {
return config.Default(), err
}
fileCfg, err := config.FromMap(file)
if err != nil {
return fileCfg, fileError(path, err)
}
cluster, ok := storedSettings(fileCfg)
if !ok {
return fileCfg, nil
}
cfg, err := config.FromMap(config.Layer(file, cluster))
if err != nil {
return fileCfg, nil //nolint:nilerr // the file still says what this machine is
}
return cfg, nil
}
// storedSettings are the cluster's settings, when the store a configuration
// names can be reached and holds some.
func storedSettings(cfg config.Server) (map[string]any, bool) {
if cfg.Auth.Store == "" {
return nil, false
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
st, err := storepkg.Open(ctx, cfg.Auth.Store)
if err != nil {
return nil, false
}
defer st.Close()
doc, err := st.ClusterSettings(ctx)
if err != nil || doc.Version == 0 {
return nil, false
}
cluster, err := config.Parse([]byte(doc.Body))
return cluster, err == nil
}
// changeStoredSettings changes the cluster's settings from a command run on
// a control plane (init), when the store holds them: changed is false when
// it does not, and the file is then where the change belongs.
func changeStoredSettings(path, note string, change func(doc *settings.Doc) error) (changed bool, err error) {
_, file, err := readSettingsFile(path)
if err != nil {
return false, err
}
fileCfg, err := config.FromMap(file)
if err != nil || fileCfg.Auth.Store == "" {
return false, nil //nolint:nilerr // no store to keep them in
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
st, err := storepkg.Open(ctx, fileCfg.Auth.Store)
if err != nil {
return false, nil //nolint:nilerr // not set up yet: the file it is
}
defer st.Close()
cur, err := st.ClusterSettings(ctx)
if err != nil || cur.Version == 0 {
return false, err
}
doc, err := settings.FromBytes("the cluster's settings", []byte(cur.Body))
if err != nil {
return false, err
}
if err := change(doc); err != nil {
return false, err
}
body, err := doc.Bytes()
if err != nil {
return false, err
}
if err := checkClusterSettings(path, body); err != nil {
return false, err
}
who := "llmgw"
if u := os.Getenv("SUDO_USER"); u != "" {
who = u
}
_, err = st.PutClusterSettings(ctx, cur.Version, string(body), who, note)
return err == nil, err
}
+139
View File
@@ -0,0 +1,139 @@
package cli
import (
"context"
"io"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func settingsFixture(t *testing.T, extra string) (path, store string) {
t.Helper()
dir := t.TempDir()
store = filepath.Join(dir, "llmgw.db")
path = filepath.Join(dir, "server.yaml")
body := "listen: \":8080\"\nagent_insecure: true\nauth:\n enabled: true\n store: " + store +
"\nrouter:\n retries: 2\n" + extra
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return path, store
}
var quiet = slog.New(slog.NewTextHandler(io.Discard, nil))
func TestTheFirstStartBringsTheFilesSettingsInWithGuardrails(t *testing.T) {
path, store := settingsFixture(t, "")
s, err := loadClusterSettings(t.Context(), path, quiet)
if err != nil {
t.Fatal(err)
}
if s.version != 1 || s.cfg.Router.Retries != 2 {
t.Errorf("version %d retries %d", s.version, s.cfg.Router.Retries)
}
if len(s.cfg.Guard.Policies) == 0 || s.cfg.Guard.Policies["baseline"].Judge != nil {
t.Errorf("a file without guard got %+v, want the baseline without a judge", s.cfg.Guard)
}
st, _ := storepkg.Open(t.Context(), store)
defer st.Close()
doc, _ := st.ClusterSettings(t.Context())
if strings.Contains(doc.Body, "listen") || strings.Contains(doc.Body, store) || !strings.Contains(doc.Body, "retries: 2") {
t.Errorf("the store holds a machine's keys, or lost the cluster's:\n%s", doc.Body)
}
// From then on the store's settings are the cluster's, whatever a
// file says.
_ = os.WriteFile(path, []byte(strings.Replace(mustRead(t, path), "retries: 2", "retries: 9", 1)), 0o600)
again, err := loadClusterSettings(t.Context(), path, quiet)
if err != nil || again.version != 1 || again.cfg.Router.Retries != 2 {
t.Errorf("second start: version %d retries %d err %v", again.version, again.cfg.Router.Retries, err)
}
}
func mustRead(t *testing.T, p string) string {
b, err := os.ReadFile(p)
if err != nil {
t.Fatal(err)
}
return string(b)
}
func TestEveryChangeIsAppliedLiveOrByStartingAgain(t *testing.T) {
path, store := settingsFixture(t, "")
s, err := loadClusterSettings(t.Context(), path, quiet)
if err != nil {
t.Fatal(err)
}
st, _ := storepkg.Open(t.Context(), store)
defer st.Close()
var lived [][]string
var restarts []string
w := &settingsWatch{path: path, st: st, cur: s, log: quiet,
live: func(_ config.Server, changed []string) bool {
lived = append(lived, changed)
return onlyLive(changed)
},
restart: func(why string) { restarts = append(restarts, why) },
}
ctx := context.Background()
put := func(body string) {
t.Helper()
v, _ := st.ClusterSettingsVersion(ctx)
if _, err := st.PutClusterSettings(ctx, v, body, "test", ""); err != nil {
t.Fatal(err)
}
}
cur, _ := st.ClusterSettings(ctx)
w.check(ctx) // nothing changed
if len(lived)+len(restarts) != 0 {
t.Fatalf("acted on no change: %v %v", lived, restarts)
}
put(strings.Replace(cur.Body, "injection: flag", "injection: block", 1))
w.check(ctx)
if len(restarts) != 0 || len(lived) != 1 || lived[0][0] != "guard" {
t.Fatalf("a guard change: lived %v restarts %v", lived, restarts)
}
cur, _ = st.ClusterSettings(ctx)
put(strings.Replace(cur.Body, "retries: 2", "retries: 4", 1))
w.check(ctx)
if len(restarts) != 1 || !strings.Contains(restarts[0], "router") || w.cur.cfg.Router.Retries != 4 {
t.Fatalf("a router change: restarts %v retries %d", restarts, w.cur.cfg.Router.Retries)
}
// A machine's own key, in its file.
_ = os.WriteFile(path, []byte(strings.Replace(mustRead(t, path), `":8080"`, `":8090"`, 1)), 0o600)
w.check(ctx)
if len(restarts) != 2 || !strings.Contains(restarts[1], "listen") {
t.Fatalf("a listener change: %v", restarts)
}
// Settings that do not load are not put in force.
cur, _ = st.ClusterSettings(ctx)
put(cur.Body + "control:\n timezone: Mars/Olympus\n")
w.check(ctx)
if len(restarts) != 2 || w.cur.cfg.Control.Timezone != "" {
t.Errorf("bad settings were applied: %v %q", restarts, w.cur.cfg.Control.Timezone)
}
}
func TestAClusterDocumentCannotSetAMachinesKeys(t *testing.T) {
path, _ := settingsFixture(t, "")
if err := checkClusterSettings(path, []byte("listen: \":1\"\n")); err == nil {
t.Error("listen was accepted")
}
if err := checkClusterSettings(path, []byte("control:\n timezone: Mars/Olympus\n")); err == nil {
t.Error("a timezone that is not one was accepted")
}
if err := checkClusterSettings(path, []byte("router:\n retries: 3\n")); err != nil {
t.Error(err)
}
}
+1
View File
@@ -173,6 +173,7 @@ func newCtlCmd() *cobra.Command {
newCapacityCmd(client, &asJSON),
newQuotaCmd(client, &asJSON),
newGuardCmd(client, &asJSON),
newConfigCmd(client, &asJSON),
newAuditCmd(client, &asJSON),
newNotifyCmd(client, &asJSON),
newUsageCmd(client, &asJSON),
+377
View File
@@ -0,0 +1,377 @@
package cli
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
func newConfigCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var version int64
c := &cobra.Command{
Use: "config",
Short: "The cluster's settings: show, change, and go back",
Long: "Everything in server.yaml that is not one machine's own — guardrails,\n" +
"limits, pricing, routing, engines, notifications, the mirror's policy —\n" +
"is the cluster's, kept in the store and the same on every control plane.\n" +
"A change is checked, kept as a new version, and in force on every control\n" +
"plane within seconds: live where it can be (guard), otherwise each control\n" +
"plane starts again on it, standbys first and the leader last.\n\n" +
"With no subcommand, prints the settings in force.\n\n" +
"A machine's own keys (" + strings.Join(config.LocalKeys(), ", ") + ")\n" +
"stay in its /etc/llmgw/server.yaml; a change there is noticed and applied\n" +
"on that machine the same way.",
Example: " llmgw ctl config\n" +
" llmgw ctl config get router.retries\n" +
" llmgw ctl config set router.retries 3 --note 'flaky upstream'\n" +
" llmgw ctl config set firewall_networks '[10.20.0.0/16]'\n" +
" llmgw ctl config edit\n" +
" llmgw ctl config history\n" +
" llmgw ctl config rollback 4",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
path := "/api/v1/settings"
if version > 0 {
path += "?version=" + strconv.FormatInt(version, 10)
}
var v adminapi.SettingsView
if err := client().get(cmd.Context(), path, &v); err != nil {
return err
}
if *asJSON {
return emitJSON(v)
}
fmt.Fprintf(cmd.ErrOrStderr(), "# version %d, %s by %s%s\n", v.Version, v.At.Local().Format(time.DateTime),
orUnknown(v.By), noteSuffix(v.Note))
fmt.Fprint(cmd.OutOrStdout(), v.Body)
return nil
},
}
c.Flags().Int64Var(&version, "version", 0, "an earlier version")
c.AddCommand(newConfigGetCmd(client), newConfigSetCmd(client), newConfigUnsetCmd(client),
newConfigEditCmd(client), newConfigHistoryCmd(client, asJSON), newConfigRollbackCmd(client))
return c
}
func orUnknown(s string) string {
if s == "" {
return "unknown"
}
return s
}
func noteSuffix(n string) string {
if n == "" {
return ""
}
return ": " + n
}
// keyPath reads a dotted key, refusing one machine's own.
func keyPath(key string) ([]string, error) {
path := strings.Split(key, ".")
if config.IsLocal(path) {
return nil, fmt.Errorf("%s is each machine's own: change it in that machine's /etc/llmgw/server.yaml, "+
"where a change is applied by itself too", key)
}
return path, nil
}
func newConfigGetCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "get KEY",
Short: "One key of the cluster's settings, as YAML",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
path, err := keyPath(args[0])
if err != nil {
return err
}
var v adminapi.SettingsView
if err := client().get(cmd.Context(), "/api/v1/settings", &v); err != nil {
return err
}
doc, err := settings.FromBytes("settings", []byte(v.Body))
if err != nil {
return err
}
n := doc.Node(path)
if n == nil {
return fmt.Errorf("%s is not set in the cluster's settings; its default applies (docs/reference.md)", args[0])
}
sub, err := settings.FromBytes("value", nil)
if err != nil {
return err
}
if err := sub.SetNode([]string{"v"}, n); err != nil {
return err
}
b, err := sub.Bytes()
if err != nil {
return err
}
out := strings.TrimPrefix(string(b), "v:")
fmt.Fprintln(cmd.OutOrStdout(), strings.TrimSpace(dedent(out)))
return nil
},
}
}
// dedent takes off the two spaces a block nested under one key carries.
func dedent(s string) string {
lines := strings.Split(s, "\n")
for i, l := range lines {
lines[i] = strings.TrimPrefix(l, " ")
}
return strings.Join(lines, "\n")
}
// changeSettings reads the newest settings, changes them, and puts the
// result in force, saying what that does.
func changeSettings(cmd *cobra.Command, client func() *ctlClient, note string,
change func(doc *settings.Doc) error,
) error {
ctx := cmd.Context()
var cur adminapi.SettingsView
if err := client().get(ctx, "/api/v1/settings", &cur); err != nil {
return err
}
doc, err := settings.FromBytes("settings", []byte(cur.Body))
if err != nil {
return err
}
if err := change(doc); err != nil {
return err
}
b, err := doc.Bytes()
if err != nil {
return err
}
return putSettings(cmd, client, cur, b, note)
}
func putSettings(cmd *cobra.Command, client func() *ctlClient, cur adminapi.SettingsView, body []byte, note string) error {
before, _ := config.Parse([]byte(cur.Body))
after, err := config.Parse(body)
if err != nil {
return err
}
changed := config.Changed(before, after)
if len(changed) == 0 {
fmt.Fprintln(cmd.OutOrStdout(), "nothing changed")
return nil
}
req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note})
var v adminapi.SettingsView
if err := client().put(cmd.Context(), "/api/v1/settings", req, &v); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "version %d: %s\n", v.Version, strings.Join(changed, ", "))
if onlyLive(changed) {
fmt.Fprintln(cmd.OutOrStdout(), "in force on every control plane within seconds")
} else {
fmt.Fprintln(cmd.OutOrStdout(), "every control plane starts again on it within seconds, standbys first, "+
"the leader half a minute later; engines keep serving")
}
return nil
}
func newConfigSetCmd(client func() *ctlClient) *cobra.Command {
var note string
c := &cobra.Command{
Use: "set KEY VALUE",
Short: "Change one key of the cluster's settings; VALUE is YAML",
Example: " llmgw ctl config set router.retries 3\n" +
" llmgw ctl config set guard.default '[baseline, finance]'\n" +
" llmgw ctl config set notify.webhooks '[{url: https://hooks.example/llmgw}]'",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
path, err := keyPath(args[0])
if err != nil {
return err
}
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
return doc.SetYAML(path, args[1])
})
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the version")
return c
}
func newConfigUnsetCmd(client func() *ctlClient) *cobra.Command {
var note string
c := &cobra.Command{
Use: "unset KEY",
Short: "Remove one key from the cluster's settings, so its default applies",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
path, err := keyPath(args[0])
if err != nil {
return err
}
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if doc.Node(path) == nil {
return fmt.Errorf("%s is not set", args[0])
}
doc.Unset(path)
return nil
})
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the version")
return c
}
func newConfigEditCmd(client func() *ctlClient) *cobra.Command {
var note string
c := &cobra.Command{
Use: "edit",
Short: "Edit the cluster's settings in $EDITOR",
Long: "Opens the settings in force in $EDITOR (vi when unset). Saved, they are\n" +
"checked and kept as a new version; a document that does not load is\n" +
"opened again with the reason, until it does or is left unchanged.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var cur adminapi.SettingsView
if err := client().get(cmd.Context(), "/api/v1/settings", &cur); err != nil {
return err
}
dir, err := os.MkdirTemp("", "llmgw-settings")
if err != nil {
return err
}
defer os.RemoveAll(dir)
file := filepath.Join(dir, "settings.yaml")
body := []byte(cur.Body)
problem := ""
for {
text := body
if problem != "" {
text = append([]byte("# NOT SAVED: "+strings.ReplaceAll(problem, "\n", " ")+"\n"), stripProblem(body)...)
}
if err := os.WriteFile(file, text, 0o600); err != nil {
return err
}
if err := runEditor(file, cmd.InOrStdin(), cmd.OutOrStdout(), cmd.ErrOrStderr()); err != nil {
return err
}
edited, err := os.ReadFile(file) //nolint:gosec // written above
if err != nil {
return err
}
edited = stripProblem(edited)
if bytes.Equal(bytes.TrimSpace(edited), bytes.TrimSpace([]byte(cur.Body))) {
fmt.Fprintln(cmd.OutOrStdout(), "nothing changed")
return nil
}
err = putSettings(cmd, client, cur, edited, note)
if err == nil {
return nil
}
if strings.Contains(err.Error(), "changed since they were read") {
// Somebody else's change came first: opening the editor
// again on the old base would only be refused again.
return fmt.Errorf("%w; your edit is not saved", err)
}
body, problem = edited, err.Error()
}
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the version")
return c
}
func stripProblem(b []byte) []byte {
if bytes.HasPrefix(b, []byte("# NOT SAVED: ")) {
if i := bytes.IndexByte(b, '\n'); i >= 0 {
return b[i+1:]
}
}
return b
}
func runEditor(file string, in io.Reader, out, errOut io.Writer) error {
editor := os.Getenv("VISUAL")
if editor == "" {
editor = os.Getenv("EDITOR")
}
if editor == "" {
editor = "vi"
}
parts := strings.Fields(editor)
c := exec.Command(parts[0], append(parts[1:], file)...) //nolint:gosec // the operator's own editor
c.Stdin, c.Stdout, c.Stderr = in, out, errOut
if err := c.Run(); err != nil {
return fmt.Errorf("%s: %w", editor, err)
}
return nil
}
func newConfigHistoryCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var limit int
c := &cobra.Command{
Use: "history",
Short: "Versions of the cluster's settings, newest first",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var list []adminapi.SettingsView
if err := client().get(cmd.Context(), "/api/v1/settings/history?limit="+strconv.Itoa(limit), &list); err != nil {
return err
}
if *asJSON {
return emitJSON(list)
}
w := tabTo(cmd.OutOrStdout())
fmt.Fprintln(w, "VERSION\tWHEN\tBY\tNOTE")
for _, v := range list {
fmt.Fprintf(w, "%d\t%s\t%s\t%s\n", v.Version, v.At.Local().Format(time.DateTime), orUnknown(v.By), v.Note)
}
return w.Flush()
},
}
c.Flags().IntVar(&limit, "limit", 20, "how many")
return c
}
func newConfigRollbackCmd(client func() *ctlClient) *cobra.Command {
var note string
c := &cobra.Command{
Use: "rollback VERSION",
Short: "Put an earlier version of the cluster's settings back in force, as a new version",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
n, err := strconv.ParseInt(args[0], 10, 64)
if err != nil || n < 1 {
return errors.New("VERSION is a number from 'llmgw ctl config history'")
}
var cur, old adminapi.SettingsView
if err := client().get(cmd.Context(), "/api/v1/settings", &cur); err != nil {
return err
}
if err := client().get(cmd.Context(), "/api/v1/settings?version="+args[0], &old); err != nil {
return err
}
if note == "" {
note = "back to version " + args[0]
}
return putSettings(cmd, client, cur, []byte(old.Body), note)
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the version")
return c
}
+3 -3
View File
@@ -167,7 +167,7 @@ func (na *nodeAdd) prepare() error {
// are not known until the node says what it has, and are not applied
// here; doctor's verdict on vLLM is a warning either way.
if o.VLLMVersion == "" {
if cfg, err := config.Load(na.serverCfg); err == nil {
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil {
o.VLLMVersion = cfg.WantedEngines()["vllm"]
if o.VLLMVersion == "" {
o.VLLMVersion = "none"
@@ -175,7 +175,7 @@ func (na *nodeAdd) prepare() error {
}
}
// The node's ports — its engines', its metrics — for the cluster only.
if cfg, err := config.Load(na.serverCfg); err == nil && !o.NoFirewall {
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && !o.NoFirewall {
from, to := cfg.EnginePorts()
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
o.FirewallAllow, _ = cfg.FirewallAllow()
@@ -183,7 +183,7 @@ func (na *nodeAdd) prepare() error {
// Where the control plane upgrades nodes from, so a node joins on the
// same release from the same place: an offline cluster's mirror.
if o.Releases == "" {
if cfg, err := config.Load(na.serverCfg); err == nil {
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil {
o.Releases = cfg.ReleasesBase()
}
if o.Releases == "" {
+80 -4
View File
@@ -16,6 +16,7 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -80,10 +81,13 @@ func newServerCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, _ []string) error {
log := logger()
cfg, err := config.Load(configPath)
// server.yaml for this machine's own keys, the store's settings
// for the cluster's: the same on every control plane.
boot, err := loadClusterSettings(cmd.Context(), configPath, log)
if err != nil {
return err
}
cfg := boot.cfg
if listen != "" {
cfg.Listen = listen
}
@@ -166,7 +170,7 @@ func newServerCmd() *cobra.Command {
}
gate.SetJudge(rt.Judge, rt.GuardFormat)
rt.SetGuard(gate)
rt.SetGuardLibrary(cfg.Guard)
rt.SetGuardLibrary(gate.Library)
if err := rt.RegisterMetrics(gate); err != nil {
return fmt.Errorf("guard metrics: %w", err)
}
@@ -175,6 +179,10 @@ func newServerCmd() *cobra.Command {
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
// Cancelled to start again on changed settings.
ctx, cancelRun := context.WithCancel(ctx)
defer cancelRun()
var restartReason atomic.Pointer[string]
rt.Start(ctx)
defer rt.Stop()
@@ -341,12 +349,16 @@ func newServerCmd() *cobra.Command {
evals = newEvalService(cfg, rt, admin, log)
api.SetEval(evals)
if st := rt.Store(); st != nil && gate != nil {
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
api.SetGuard(&adminapi.GuardAdmin{Library: gate.Library, Store: st, Invalidate: gate.Invalidate})
}
if st := rt.Store(); st != nil {
// Removing a node revokes its certificate.
api.SetRevoker(st.RevokeNode)
api.SetAudit(st)
// The cluster's settings, each change checked against this
// machine's own keys before it is kept.
api.SetSettings(&adminapi.Settings{Store: st, LocalKeys: config.LocalKeys(),
Check: func(body []byte) error { return checkClusterSettings(configPath, body) }})
}
// Catalog changes need somewhere to persist, so this is wired
// only when there is a store. Without one, the endpoints say so
@@ -378,6 +390,17 @@ func newServerCmd() *cobra.Command {
} else {
admin.ReplaceCatalog(ctx, built)
}
// And again on becoming the leader: models added or
// removed through the leader before are in the store.
if l := admin.Leadership(); l != nil {
l.OnLead(func() {
if built, err := svc.Load(ctx); err != nil {
log.Error("could not read the stored models on becoming the leader", "err", err)
} else {
admin.ReplaceCatalog(ctx, built)
}
})
}
api.SetModels(svc)
catalogSvc = svc
// And the mirror page, so a model that has just been
@@ -627,6 +650,46 @@ func newServerCmd() *cobra.Command {
}
}
// Every change to the settings in force: what can change live
// does, the rest starts this process again, which systemd brings
// back on them.
watch := &settingsWatch{path: configPath, st: rt.Store(), cur: boot, log: log,
live: func(next config.Server, changed []string) bool {
if !onlyLive(changed) {
return false
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "guard") }) {
if gate == nil {
// Built at start, and only when there are policies.
return len(next.Guard.Policies) == 0
}
if err := gate.Replace(next.Guard); err != nil {
log.Error("the changed guard policies are not valid; the old ones stay in force", "err", err)
}
}
return true
},
restart: func(reason string) {
if os.Getenv("INVOCATION_ID") == "" {
log.Warn("the settings changed in a way this process applies only when it starts; "+
"it is not run by systemd, so start it again by hand", "why", reason)
return
}
// The leader last, so its standbys are already on the new
// settings when it hands over to one of them.
delay := time.Duration(0)
if admin != nil && admin.Leadership() != nil && admin.Leadership().IsLeader() && rt.Store() != nil {
if cps, err := rt.Store().ControlPlanes(ctx); err == nil && len(cps) > 1 {
delay = 30 * time.Second
}
}
log.Warn("starting again to apply the settings", "why", reason, "in", delay)
restartReason.Store(&reason)
time.AfterFunc(delay, cancelRun)
},
}
go watch.run(ctx)
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error { return serve(apiSrv, apiLn) })
g.Go(func() error { return serve(adminSrv, adminLn) })
@@ -658,7 +721,12 @@ func newServerCmd() *cobra.Command {
_ = adminSrv.Shutdown(sctx)
return apiSrv.Shutdown(sctx)
})
return g.Wait()
err = g.Wait()
if r := restartReason.Load(); r != nil {
// Not a failure, but said as one, so systemd starts it again.
return fmt.Errorf("stopped to start again on changed settings (%s)", *r)
}
return err
},
}
@@ -756,6 +824,14 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log))
// Domains held for maintenance and labels set on nodes.
ctl.Facility().UseStore(ctx, st)
// A standby read these when it started; the leader before it may
// have changed them since.
lead.OnLead(func() {
if ch := ctl.EngineChoices(); ch != nil {
ch.Reload(ctx)
}
ctl.Facility().Reload(ctx)
})
log.Info("competing to be the control plane that places work",
"owner", lead.Owner())
}
+1 -1
View File
@@ -155,7 +155,7 @@ func (sa *serverAdd) run(ctx context.Context) error {
// firewall keeps the new control plane's internal ports to the networks it
// reaches this one on. Not fatal: it serves without one.
func (sa *serverAdd) firewall(ctx context.Context) error {
cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml"))
cfg, err := loadEffectiveConfig(filepath.Join(sa.etcDir, "server.yaml"))
if err != nil {
return err
}
+2 -14
View File
@@ -144,9 +144,7 @@ func newServerInitCmd() *cobra.Command {
"SQLite or PostgreSQL, is moved in. Run again, it changes nothing.\n\n" +
"Each step is skipped when its result already exists, so it is safe to run\n" +
"again: to finish an install that stopped, or with another --host to\n" +
"reissue the certificate. It never overwrites server.yaml or the authority;\n" +
"to a server.yaml from before guardrails it adds the guard section, the old\n" +
"file kept beside it.\n\n" +
"reissue the certificate. It never overwrites server.yaml or the authority.\n\n" +
"The authority's key stays in " + defaultPKIDir + " on this machine. Move it\n" +
"somewhere safer once the nodes are onboarded if this machine is exposed.",
Args: cobra.NoArgs,
@@ -290,7 +288,7 @@ func (in *serverInit) judgeWanted() bool {
// configuration written before init installed one is never changed, so the
// model is not fetched for nothing.
func (in *serverInit) configJudgesWith(model string) bool {
cfg, err := config.Load(in.configPath())
cfg, err := loadEffectiveConfig(in.configPath())
if err != nil {
return false
}
@@ -550,16 +548,6 @@ func (in *serverInit) ensureConfig(context.Context) error {
path := in.configPath()
if _, err := os.Stat(path); err == nil {
in.kept("%s (never overwritten)", path)
// A file from before guardrails has no guard: at all, and an
// upgrade alone leaves it unguarded. One that says guard:, even
// empty, was decided on and stays as it is.
added, err := appendGuardIfMissing(path, in.judgeWanted())
if err != nil {
return fmt.Errorf("%s: %w", path, err)
}
if added {
in.done("server.yaml: guardrails added (baseline policy), the old file kept beside it")
}
} else {
contact := in.admin
if contact == "" {
+21 -2
View File
@@ -12,6 +12,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -60,7 +61,7 @@ func (in *serverInit) offline() bool {
if in.bundle != "" {
return true
}
cfg, err := config.Load(in.configPath())
cfg, err := loadEffectiveConfig(in.configPath())
return err == nil && cfg.Offline.Enabled()
}
@@ -76,7 +77,7 @@ func (in *serverInit) mirrorRoot() string {
// server.yaml that the cluster is offline. With a mirror on another
// machine, the files are imported there; here only the checksums are kept.
func (in *serverInit) ensureBundle(ctx context.Context) error {
cfg, err := config.Load(in.configPath())
cfg, err := loadEffectiveConfig(in.configPath())
if err != nil {
return err
}
@@ -129,6 +130,24 @@ func (in *serverInit) ensureBundle(ctx context.Context) error {
if sums == config.DefaultOfflineChecksums {
sums = ""
}
// In the cluster's settings once the store holds them; in server.yaml
// before, where the first start brings it in.
stored, err := changeStoredSettings(in.configPath(), "offline, from server init --bundle", func(doc *settings.Doc) error {
if err := doc.SetYAML([]string{"offline", "mirror"}, files); err != nil {
return err
}
if sums != "" {
return doc.SetYAML([]string{"offline", "checksums"}, sums)
}
return nil
})
if err != nil {
return err
}
if stored {
in.done("the cluster's settings: offline, nodes install from the mirror at %s", files)
return nil
}
if err := appendOfflineBlock(in.configPath(), files, sums); err != nil {
return err
}
-38
View File
@@ -13,8 +13,6 @@ import (
"strings"
"time"
"go.yaml.in/yaml/v3"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
@@ -667,39 +665,3 @@ func waitFor(ctx context.Context, limit time.Duration, what string, done func()
}
}
}
// appendGuardIfMissing adds init's guard section to a server.yaml that has
// no guard key, keeping the file as it was beside it.
func appendGuardIfMissing(path string, judge bool) (bool, error) {
b, err := os.ReadFile(path) //nolint:gosec // server.yaml
if err != nil {
return false, err
}
var top map[string]any
if err := yaml.Unmarshal(b, &top); err != nil {
return false, err
}
if _, ok := top["guard"]; ok {
return false, nil
}
if err := appendBlock(path, b, guardSection(judge)); err != nil {
return false, err
}
return true, nil
}
// appendBlock writes b plus block to path, with b kept in a dated backup.
func appendBlock(path string, b []byte, block string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
backup := fmt.Sprintf("%s.bak-%s", path, time.Now().Format("20060102-150405"))
if err := os.WriteFile(backup, b, 0o640); err != nil {
return err
}
if len(b) > 0 && b[len(b)-1] != '\n' {
b = append(b, '\n')
}
return os.WriteFile(path, append(append(b, '\n'), block...), info.Mode().Perm())
}
-37
View File
@@ -207,40 +207,3 @@ func TestInitSaysAtOnceWhenAModelFitsNowhere(t *testing.T) {
t.Errorf("took %s to say so", time.Since(start))
}
}
func TestAnOldConfigurationGetsGuardrailsOnce(t *testing.T) {
body := fmt.Sprintf(serverConfigTemplate, "/etc/llmgw", "/etc/llmgw", "/etc/llmgw",
"/var/lib/llmgw/llmgw.db", "ops@example.com", "")
path := filepath.Join(t.TempDir(), "server.yaml")
if err := os.WriteFile(path, []byte(body), 0o640); err != nil {
t.Fatal(err)
}
added, err := appendGuardIfMissing(path, true)
if err != nil || !added {
t.Fatalf("added=%v err=%v, want the guard section added", added, err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatal(err)
}
if err := cfg.Validate(); err != nil {
t.Fatal(err)
}
if len(cfg.Guard.Default) != 1 || cfg.Guard.Policies["baseline"].Judge == nil {
t.Errorf("guard = %+v, want the baseline with its judge", cfg.Guard)
}
if backups, _ := filepath.Glob(path + ".bak-*"); len(backups) != 1 {
t.Errorf("backups = %v, want the old file kept", backups)
}
if added, err := appendGuardIfMissing(path, true); err != nil || added {
t.Errorf("second run: added=%v err=%v, want nothing changed", added, err)
}
// guard: written by hand, even empty, is a decision and stays.
if err := os.WriteFile(path, []byte(body+"guard: {}\n"), 0o640); err != nil {
t.Fatal(err)
}
if added, err := appendGuardIfMissing(path, true); err != nil || added {
t.Errorf("empty guard: added=%v err=%v, want it left alone", added, err)
}
}
+1 -1
View File
@@ -19,7 +19,7 @@ import (
// the engines and etcd when they are here. The client API is not among
// them; it is meant to be reached.
func (in *serverInit) ensureFirewall(ctx context.Context) error {
cfg, err := config.Load(in.configPath())
cfg, err := loadEffectiveConfig(in.configPath())
if err != nil {
return err
}
+2 -4
View File
@@ -126,10 +126,8 @@ func TestInitNeverOverwritesTheConfiguration(t *testing.T) {
if err := in.run(context.Background()); err != nil {
t.Fatalf("%v\n%s", err, out)
}
// Kept as written; only the guardrails it predates are added after it.
got, _ := os.ReadFile(in.configPath())
if !strings.HasPrefix(string(got), mine) || !strings.Contains(string(got[len(mine):]), "\nguard:\n") {
t.Errorf("server.yaml is not the old file with guard: after it:\n%s", got)
if got, _ := os.ReadFile(in.configPath()); string(got) != mine {
t.Error("an existing server.yaml was rewritten")
}
// And the store it names is the one used.
if _, err := os.Stat(filepath.Join(in.stateDir, "mine.db")); err != nil {
+81 -25
View File
@@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
@@ -138,12 +139,15 @@ type consoleModel struct {
node bool
// The settings.
file string
doc *settings.Doc
cfg any // the file as loaded, for the values that apply
levels []settingsLevel
docErr error
saveNote string
file string
doc *settings.Doc
cfg any // the file as loaded, for the values that apply
// settingsBase is the version of the cluster's settings the screen
// read, when the cluster keeps them (doc is then layered over them).
settingsBase int64
levels []settingsLevel
docErr error
saveNote string
// A question open as a dialog.
dialog string // "", "target", "edit", "addname", "saved", "confirm-leave", "error"
@@ -219,6 +223,26 @@ func (m *consoleModel) loadSettings() {
} else if !m.node {
m.cfg = config.Default()
}
// The cluster's keys from the cluster's settings, when the control
// plane answers with them; its own keys from its file.
if !m.node && m.client != nil {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
var v adminapi.SettingsView
if err := m.client.get(ctx, "/api/v1/settings", &v); err == nil && v.Version > 0 {
if cdoc, err := settings.FromBytes("the cluster's settings", []byte(v.Body)); err == nil {
m.doc.Layer(cdoc, config.IsLocal)
m.settingsBase = v.Version
if file, err := config.ReadFile(m.file); err == nil {
if cluster, err := config.Parse([]byte(v.Body)); err == nil {
if cfg, err := config.FromMap(config.Layer(file, cluster)); err == nil {
m.cfg = cfg
}
}
}
}
}
}
if root, err := settings.Root(m.file); err == nil && len(m.levels) == 0 {
m.levels = []settingsLevel{{title: filepathBase(m.file), block: root, kind: settings.Block}}
}
@@ -643,10 +667,10 @@ func (m consoleModel) peopleView() bool {
}
func (m consoleModel) leave() (tea.Model, tea.Cmd) {
if m.doc != nil && m.doc.Changed > 0 {
if m.doc != nil && m.doc.Pending() > 0 {
m.dialog, m.dialogTitle = "confirm-leave", "Unsaved changes"
m.dialogText = fmt.Sprintf("%d change(s) to %s are not saved.\nLeave without saving them?\n\n%s",
m.doc.Changed, m.file, sDim.Render("Y leave · N stay"))
m.dialogText = fmt.Sprintf("%d change(s) are not saved.\nLeave without saving them?\n\n%s",
m.doc.Pending(), sDim.Render("Y leave · N stay"))
return m, nil
}
return m, tea.Quit
@@ -663,7 +687,7 @@ func (m consoleModel) ask(title, question string, n int) (tea.Model, tea.Cmd) {
func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
switch m.dialog {
case "error":
case "error", "note":
m.dialog = ""
return m, nil
case "confirm-leave":
@@ -917,28 +941,60 @@ func (m consoleModel) settingsKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
}
func (m consoleModel) save() (tea.Model, tea.Cmd) {
if m.doc.Changed == 0 {
if m.doc.Pending() == 0 {
m.dialog, m.dialogTitle, m.dialogText = "error", "Nothing to save", "No change has been made."
return m, nil
}
backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err })
if err != nil {
notSaved := func(err error) (tea.Model, tea.Cmd) {
m.dialog, m.dialogTitle = "error", "Not saved"
m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" + sDim.Render("Nothing was written. Any key to go back.")
m.dialogText = "The configuration would not load:\n\n" + sBad.Render(err.Error()) + "\n\n" +
sDim.Render("Nothing was written. Any key to go back.")
return m, nil
}
var said []string
// The cluster's settings first: checked by the control plane against
// its own file, and in force everywhere once kept.
if c := m.doc.Cluster(); c != nil && c.Changed > 0 {
body, err := c.Bytes()
if err != nil {
return notSaved(err)
}
req, _ := json.Marshal(adminapi.SettingsRequest{Base: m.settingsBase, Body: string(body), Note: "console"})
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var v adminapi.SettingsView
if err := m.client.put(ctx, "/api/v1/settings", req, &v); err != nil {
return notSaved(err)
}
c.Changed = 0
said = append(said, fmt.Sprintf("The cluster's settings are version %d: in force on every control plane "+
"within seconds.\nllmgw ctl config history lists the versions; rollback goes back to one.", v.Version))
}
fileSaved := m.doc.Changed > 0
if fileSaved {
backup, err := m.doc.Save(func(p string) error { _, err := m.load(p); return err })
if err != nil {
return notSaved(err)
}
line := "Saved " + m.file + "."
if backup != "" {
line += " The old file is " + backup + "."
}
said = append(said, line)
}
m.loadSettings()
note := ""
if backup != "" {
note = "The old file is " + backup + ".\n"
if !m.node {
if fileSaved {
said = append(said, "llmgw-server notices the change and applies it within seconds.")
}
m.dialog, m.dialogTitle = "note", "Saved"
m.dialogText = strings.Join(said, "\n\n") + "\n\n" + sDim.Render("Any key to go back.")
return m, nil
}
m.dialog, m.dialogTitle = "saved", "Saved"
m.dialogText = "Saved " + m.file + ".\n" + note +
fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service())
if !m.node {
m.dialogText += sDim.Render("Another control plane keeps its own copy: make the change there too.") + "\n"
}
m.dialogText += "\n" + sDim.Render("Y restart · N later")
m.dialogText = strings.Join(said, "\n") + "\n" +
fmt.Sprintf("\n%s reads it when it starts. Restart it now?\n", m.service()) +
"\n" + sDim.Render("Y restart · N later")
return m, nil
}
@@ -1011,8 +1067,8 @@ func (m consoleModel) View() string {
if n := runningJobs(m.data.jobs); i == cJobs && n > 0 {
label += fmt.Sprintf(" (%d)", n)
}
if i == cSettings && m.doc != nil && m.doc.Changed > 0 {
label += fmt.Sprintf(" (%d)", m.doc.Changed)
if i == cSettings && m.doc != nil && m.doc.Pending() > 0 {
label += fmt.Sprintf(" (%d)", m.doc.Pending())
}
nav = append(nav, navItem{Label: label, Current: i == m.menu, Dim: !m.entryOK(i)})
}
+1 -2
View File
@@ -13,7 +13,6 @@ import (
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
@@ -57,7 +56,7 @@ var hfClients sync.Map
func (m consoleModel) registry() *hf.Client {
endpoint := ""
if cfg, err := config.Load(m.file); err == nil {
if cfg, err := loadEffectiveConfig(m.file); err == nil {
endpoint = cfg.Mirror.Registry
}
c, _ := hfClients.LoadOrStore(endpoint, hf.New(endpoint, 20*time.Second))
+1 -2
View File
@@ -12,7 +12,6 @@ import (
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -118,7 +117,7 @@ func (m consoleModel) fetchPeople() tea.Cmd {
}
}
}
if cfg, err := config.Load(file); err == nil {
if cfg, err := loadEffectiveConfig(file); err == nil {
for name := range cfg.Guard.Policies {
d.guardPolicies = append(d.guardPolicies, name)
}
+57 -1
View File
@@ -1,6 +1,9 @@
package cli
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
@@ -164,7 +167,7 @@ func TestTheConsoleWatchesTheClusterAndEditsItsSettings(t *testing.T) {
cm = mm.(consoleModel)
cm.prompt.SetValue("5")
mm = press(cm, "enter", "s")
if d := mm.(consoleModel).dialog; d != "saved" {
if d := mm.(consoleModel).dialog; d != "note" {
t.Fatalf("after saving the dialog is %q: %s", d, mm.(consoleModel).dialogText)
}
b, _ := os.ReadFile(file)
@@ -244,3 +247,56 @@ func TestTheConsoleShowsTheGuardrails(t *testing.T) {
t.Errorf("window %q", guardWindows[c.guard.window])
}
}
func TestTheConsoleEditsTheClustersSettingsInTheStore(t *testing.T) {
dir := t.TempDir()
file := filepath.Join(dir, "server.yaml")
if err := os.WriteFile(file, []byte("listen: \":8080\"\n"), 0o644); err != nil {
t.Fatal(err)
}
var put adminapi.SettingsRequest
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == "GET" && r.URL.Path == "/api/v1/settings":
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 7, Body: "router:\n retries: 2\n"})
case r.Method == "PUT" && r.URL.Path == "/api/v1/settings":
_ = json.NewDecoder(r.Body).Decode(&put)
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 8, Body: put.Body})
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
m := newConsoleModel(strings.TrimPrefix(srv.URL, "http://"), file)
m.width, m.height, m.loaded = 120, 40, true
m.menu, m.inPanel = cSettings, true
m.loadSettings()
for i, r := range m.settingsRows() {
if r.label == "router" {
m.row = i
}
}
var mm tea.Model = press(m, "enter")
cm := mm.(consoleModel)
for i, r := range cm.settingsRows() {
if r.label == "retries" {
cm.row = i
}
}
if v := cm.View(); !strings.Contains(v, "2") {
t.Errorf("the cluster's value is not shown:\n%s", v)
}
mm = press(cm, "enter")
cm = mm.(consoleModel)
cm.prompt.SetValue("5")
mm = press(cm, "enter", "s")
if d := mm.(consoleModel).dialog; d != "note" || !strings.Contains(mm.(consoleModel).dialogText, "version 8") {
t.Fatalf("after saving: %q %s", d, mm.(consoleModel).dialogText)
}
if put.Base != 7 || !strings.Contains(put.Body, "retries: 5") {
t.Errorf("sent %+v", put)
}
if b, _ := os.ReadFile(file); string(b) != "listen: \":8080\"\n" {
t.Errorf("the cluster's key went into the machine's file:\n%s", b)
}
}