llmgw/internal/cli/clustersettings.go

package cli

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"slices"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
	"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
	storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// settingsPoll is how often a control plane asks whether the cluster's
// settings, or its own server.yaml, changed.
const settingsPoll = 5 * time.Second

// clusterSettings is what a control plane runs on, and where it came from:
// its server.yaml for its own keys, the store's settings for the rest.
type clusterSettings struct {
	cfg      config.Server
	eff      map[string]any // the settings, as written, cfg was made from
	cluster  map[string]any // the store's document, nil without a store
	version  int64
	fileHash [32]byte
}

// loadClusterSettings reads the configuration a control plane starts on.
// Without a store it is the file. With one, the store's settings take the
// place of the file's cluster keys; a store with none yet is given the
// file's, so the first control plane to start against it brings them in.
func loadClusterSettings(ctx context.Context, path string, log *slog.Logger) (clusterSettings, error) {
	var s clusterSettings
	b, file, err := readSettingsFile(path)
	if err != nil {
		return s, err
	}
	s.fileHash = sha256.Sum256(b)
	fileCfg, err := config.FromMap(file)
	if err != nil {
		return s, fileError(path, err)
	}
	if fileCfg.Auth.Store == "" {
		s.cfg, s.eff = fileCfg, file
		return s, nil
	}
	st, err := storepkg.Open(ctx, fileCfg.Auth.Store)
	if err != nil {
		return s, fmt.Errorf("the store, for the cluster's settings: %w", err)
	}
	defer st.Close()

	doc, err := st.ClusterSettings(ctx)
	if err != nil {
		return s, fmt.Errorf("reading the cluster's settings: %w", err)
	}
	if doc.Version == 0 {
		if doc, err = importSettings(ctx, st, path, file, log); err != nil {
			return s, err
		}
	} else if ignored := config.Ignored(file); len(ignored) > 0 {
		log.Warn("server.yaml sets keys the cluster's settings now hold; they are not read "+
			"(llmgw ctl config shows what is in force)", "keys", strings.Join(ignored, ", "))
	}
	// The newest version that this machine can run on. One that cannot — a
	// path only another control plane has — must not keep this one down,
	// where nothing could be used to put it right.
	versions := []storepkg.SettingsVersion{doc}
	if older, err := st.ClusterSettingsHistory(ctx, 20); err == nil && len(older) > 1 {
		versions = append(versions, older[1:]...)
	}
	var firstErr error
	for _, v := range versions {
		cluster, err := config.Parse([]byte(v.Body))
		if err == nil {
			eff := config.Layer(file, cluster)
			var cfg config.Server
			if cfg, err = config.FromMap(eff); err == nil {
				if v.Version != doc.Version {
					log.Error("the cluster's newest settings do not load on this machine; running on an older version",
						"newest", doc.Version, "running", v.Version, "err", firstErr)
				}
				s.cfg, s.eff, s.cluster, s.version = cfg, eff, cluster, doc.Version
				return s, nil
			}
		}
		if firstErr == nil {
			firstErr = err
		}
	}
	return s, fmt.Errorf("the cluster's settings (version %d) do not load on this machine: %w", doc.Version, firstErr)
}

// importSettings records a server.yaml's cluster keys as the cluster's first
// settings, with the baseline guard policy when the file has none: a cluster
// installed before guardrails gets them on its first start after an upgrade.
func importSettings(ctx context.Context, st *storepkg.DB, path string, file map[string]any,
	log *slog.Logger,
) (storepkg.SettingsVersion, error) {
	_, cluster := config.Split(file)
	note := "imported from " + path
	if _, ok := cluster["guard"]; !ok {
		baseline, err := config.Parse([]byte(guardSection(false)))
		if err == nil {
			cluster["guard"] = baseline["guard"]
			note += ", with the baseline guard policy it did not have"
		}
	}
	body, err := config.Marshal(cluster)
	if err != nil {
		return storepkg.SettingsVersion{}, err
	}
	host, _ := os.Hostname()
	_, err = st.PutClusterSettings(ctx, 0, string(body), host, note)
	switch {
	case errors.Is(err, storepkg.ErrSettingsChanged):
		// Another control plane brought its own in first: those are the
		// cluster's.
	case err != nil:
		return storepkg.SettingsVersion{}, fmt.Errorf("recording the cluster's settings: %w", err)
	default:
		log.Info("the cluster's settings are now kept in the store; server.yaml keeps this machine's own",
			"from", path)
	}
	return st.ClusterSettings(ctx)
}

func readSettingsFile(path string) ([]byte, map[string]any, error) {
	if path == "" {
		return nil, map[string]any{}, nil
	}
	b, err := os.ReadFile(path) //nolint:gosec // the configuration
	if err != nil {
		return nil, nil, fmt.Errorf("config file: %w", err)
	}
	m, err := config.Parse(b)
	if err != nil {
		return nil, nil, fmt.Errorf("parsing %s: %w", path, err)
	}
	return b, m, nil
}

func fileError(path string, err error) error {
	if path == "" {
		return err
	}
	return fmt.Errorf("%s: %w", path, err)
}

// settingsWatch follows the cluster's settings and this machine's
// server.yaml, and puts every change in force.
type settingsWatch struct {
	path string
	st   *storepkg.DB // nil: the file only
	cur  clusterSettings
	// live applies a change without a restart, reporting whether it could.
	live func(cfg config.Server, changed []string) bool
	// restart has the process start again on the new settings.
	restart func(reason string)
	log     *slog.Logger
}

func (w *settingsWatch) run(ctx context.Context) {
	t := time.NewTicker(settingsPoll)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			w.check(ctx)
		}
	}
}

// check reads what changed, if anything, and applies it.
func (w *settingsWatch) check(ctx context.Context) {
	version := w.cur.version
	if w.st != nil {
		v, err := w.st.ClusterSettingsVersion(ctx)
		if err != nil {
			return // asked again in a few seconds
		}
		version = v
	}
	b, file, err := readSettingsFile(w.path)
	if err != nil {
		w.log.Error("server.yaml cannot be read; running on what it said before", "err", err)
		return
	}
	hash := sha256.Sum256(b)
	if version == w.cur.version && hash == w.cur.fileHash {
		return
	}
	cluster := w.cur.cluster
	eff := file
	if w.st != nil {
		if version != w.cur.version {
			doc, err := w.st.ClusterSettings(ctx)
			if err != nil {
				return
			}
			if cluster, err = config.Parse([]byte(doc.Body)); err != nil {
				w.log.Error("the cluster's settings do not parse; running on what was in force",
					"version", doc.Version, "err", err)
				w.cur.version, w.cur.fileHash = version, hash
				return
			}
		}
		eff = config.Layer(file, cluster)
	}
	cfg, err := config.FromMap(eff)
	if err != nil {
		w.log.Error("the changed settings do not load on this machine; running on what was in force",
			"version", version, "err", err)
		w.cur.version, w.cur.fileHash = version, hash
		return
	}
	changed := config.Changed(w.cur.eff, eff)
	w.cur = clusterSettings{cfg: cfg, eff: eff, cluster: cluster, version: version, fileHash: hash}
	if len(changed) == 0 {
		return
	}
	if w.live != nil && w.live(cfg, changed) {
		w.log.Info("settings changed and in force", "version", version, "changed", strings.Join(changed, ", "))
		return
	}
	w.restart(fmt.Sprintf("settings version %d changed %s", version, strings.Join(changed, ", ")))
}

// liveSettings are the keys a control plane puts in force without starting
// again: what it reads at every use, or not at all.
var liveSettings = []string{
	"guard",
	// Stopping and starting the service (llmgw stop, llmgw start).
	"serving",
	// Sent to the nodes and this machine's firewall unit as they change.
	"firewall", "firewall_networks",
	// Sent to the nodes' agents as they change.
	"agents",
	// Read by server init, node add and backups, not by the server.
	"backup_recipients", "backup_copy_to",
}

// onlyLive is whether every changed key is one of liveSettings.
func onlyLive(changed []string) bool {
	for _, c := range changed {
		top, _, _ := strings.Cut(c, ".")
		if !slices.Contains(liveSettings, top) {
			return false
		}
	}
	return true
}

// checkClusterSettings refuses a settings document that would not load,
// laid over the server.yaml at path, or that sets one machine's keys.
func checkClusterSettings(path string, body []byte) error {
	cluster, err := config.Parse(body)
	if err != nil {
		return err
	}
	if err := config.CheckCluster(cluster); err != nil {
		return err
	}
	_, file, err := readSettingsFile(path)
	if err != nil {
		return err
	}
	_, err = config.FromMap(config.Layer(file, cluster))
	return err
}

// loadEffectiveConfig is what the control plane on this machine runs on:
// its server.yaml with the cluster's settings from the store laid over it,
// for the commands beside it that read cluster keys — init, node add,
// backups. The file alone when the store cannot be reached or holds none.
func loadEffectiveConfig(path string) (config.Server, error) {
	_, file, err := readSettingsFile(path)
	if err != nil {
		return config.Default(), err
	}
	fileCfg, err := config.FromMap(file)
	if err != nil {
		return fileCfg, fileError(path, err)
	}
	cluster, ok := storedSettings(fileCfg)
	if !ok {
		return fileCfg, nil
	}
	cfg, err := config.FromMap(config.Layer(file, cluster))
	if err != nil {
		return fileCfg, nil //nolint:nilerr // the file still says what this machine is
	}
	return cfg, nil
}

// storedSettings are the cluster's settings, when the store a configuration
// names can be reached and holds some.
func storedSettings(cfg config.Server) (map[string]any, bool) {
	if cfg.Auth.Store == "" {
		return nil, false
	}
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	st, err := storepkg.Open(ctx, cfg.Auth.Store)
	if err != nil {
		return nil, false
	}
	defer st.Close()
	doc, err := st.ClusterSettings(ctx)
	if err != nil || doc.Version == 0 {
		return nil, false
	}
	cluster, err := config.Parse([]byte(doc.Body))
	return cluster, err == nil
}

// changeStoredSettings changes the cluster's settings from a command run on
// a control plane (init), when the store holds them: changed is false when
// it does not, and the file is then where the change belongs.
func changeStoredSettings(path, note string, change func(doc *settings.Doc) error) (changed bool, err error) {
	_, file, err := readSettingsFile(path)
	if err != nil {
		return false, err
	}
	fileCfg, err := config.FromMap(file)
	if err != nil || fileCfg.Auth.Store == "" {
		return false, nil //nolint:nilerr // no store to keep them in
	}
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	st, err := storepkg.Open(ctx, fileCfg.Auth.Store)
	if err != nil {
		return false, nil //nolint:nilerr // not set up yet: the file it is
	}
	defer st.Close()
	cur, err := st.ClusterSettings(ctx)
	if err != nil || cur.Version == 0 {
		return false, err
	}
	doc, err := settings.FromBytes("the cluster's settings", []byte(cur.Body))
	if err != nil {
		return false, err
	}
	if err := change(doc); err != nil {
		return false, err
	}
	body, err := doc.Bytes()
	if err != nil {
		return false, err
	}
	if err := checkClusterSettings(path, body); err != nil {
		return false, err
	}
	who := "llmgw"
	if u := os.Getenv("SUDO_USER"); u != "" {
		who = u
	}
	_, err = st.PutClusterSettings(ctx, cur.Version, string(body), who, note)
	return err == nil, err
}