package cli import ( "context" "crypto/sha256" "errors" "fmt" "log/slog" "os" "slices" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // settingsPoll is how often a control plane asks whether the cluster's // settings, or its own server.yaml, changed. const settingsPoll = 5 * time.Second // clusterSettings is what a control plane runs on, and where it came from: // its server.yaml for its own keys, the store's settings for the rest. type clusterSettings struct { cfg config.Server eff map[string]any // the settings, as written, cfg was made from cluster map[string]any // the store's document, nil without a store version int64 fileHash [32]byte } // loadClusterSettings reads the configuration a control plane starts on. // Without a store it is the file. With one, the store's settings take the // place of the file's cluster keys; a store with none yet is given the // file's, so the first control plane to start against it brings them in. func loadClusterSettings(ctx context.Context, path string, log *slog.Logger) (clusterSettings, error) { var s clusterSettings b, file, err := readSettingsFile(path) if err != nil { return s, err } s.fileHash = sha256.Sum256(b) fileCfg, err := config.FromMap(file) if err != nil { return s, fileError(path, err) } if fileCfg.Auth.Store == "" { s.cfg, s.eff = fileCfg, file return s, nil } st, err := storepkg.Open(ctx, fileCfg.Auth.Store) if err != nil { return s, fmt.Errorf("the store, for the cluster's settings: %w", err) } defer st.Close() doc, err := st.ClusterSettings(ctx) if err != nil { return s, fmt.Errorf("reading the cluster's settings: %w", err) } if doc.Version == 0 { if doc, err = importSettings(ctx, st, path, file, log); err != nil { return s, err } } else if ignored := config.Ignored(file); len(ignored) > 0 { log.Warn("server.yaml sets keys the cluster's settings now hold; they are not read "+ "(llmgw ctl config shows what is in force)", "keys", strings.Join(ignored, ", ")) } // The newest version that this machine can run on. One that cannot — a // path only another control plane has — must not keep this one down, // where nothing could be used to put it right. versions := []storepkg.SettingsVersion{doc} if older, err := st.ClusterSettingsHistory(ctx, 20); err == nil && len(older) > 1 { versions = append(versions, older[1:]...) } var firstErr error for _, v := range versions { cluster, err := config.Parse([]byte(v.Body)) if err == nil { eff := config.Layer(file, cluster) var cfg config.Server if cfg, err = config.FromMap(eff); err == nil { if v.Version != doc.Version { log.Error("the cluster's newest settings do not load on this machine; running on an older version", "newest", doc.Version, "running", v.Version, "err", firstErr) } s.cfg, s.eff, s.cluster, s.version = cfg, eff, cluster, doc.Version return s, nil } } if firstErr == nil { firstErr = err } } return s, fmt.Errorf("the cluster's settings (version %d) do not load on this machine: %w", doc.Version, firstErr) } // importSettings records a server.yaml's cluster keys as the cluster's first // settings, with the baseline guard policy when the file has none: a cluster // installed before guardrails gets them on its first start after an upgrade. func importSettings(ctx context.Context, st *storepkg.DB, path string, file map[string]any, log *slog.Logger, ) (storepkg.SettingsVersion, error) { _, cluster := config.Split(file) note := "imported from " + path if _, ok := cluster["guard"]; !ok { baseline, err := config.Parse([]byte(guardSection(false))) if err == nil { cluster["guard"] = baseline["guard"] note += ", with the baseline guard policy it did not have" } } body, err := config.Marshal(cluster) if err != nil { return storepkg.SettingsVersion{}, err } host, _ := os.Hostname() _, err = st.PutClusterSettings(ctx, 0, string(body), host, note) switch { case errors.Is(err, storepkg.ErrSettingsChanged): // Another control plane brought its own in first: those are the // cluster's. case err != nil: return storepkg.SettingsVersion{}, fmt.Errorf("recording the cluster's settings: %w", err) default: log.Info("the cluster's settings are now kept in the store; server.yaml keeps this machine's own", "from", path) } return st.ClusterSettings(ctx) } func readSettingsFile(path string) ([]byte, map[string]any, error) { if path == "" { return nil, map[string]any{}, nil } b, err := os.ReadFile(path) //nolint:gosec // the configuration if err != nil { return nil, nil, fmt.Errorf("config file: %w", err) } m, err := config.Parse(b) if err != nil { return nil, nil, fmt.Errorf("parsing %s: %w", path, err) } return b, m, nil } func fileError(path string, err error) error { if path == "" { return err } return fmt.Errorf("%s: %w", path, err) } // settingsWatch follows the cluster's settings and this machine's // server.yaml, and puts every change in force. type settingsWatch struct { path string st *storepkg.DB // nil: the file only cur clusterSettings // live applies a change without a restart, reporting whether it could. live func(cfg config.Server, changed []string) bool // restart has the process start again on the new settings. restart func(reason string) log *slog.Logger } func (w *settingsWatch) run(ctx context.Context) { t := time.NewTicker(settingsPoll) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: w.check(ctx) } } } // check reads what changed, if anything, and applies it. func (w *settingsWatch) check(ctx context.Context) { version := w.cur.version if w.st != nil { v, err := w.st.ClusterSettingsVersion(ctx) if err != nil { return // asked again in a few seconds } version = v } b, file, err := readSettingsFile(w.path) if err != nil { w.log.Error("server.yaml cannot be read; running on what it said before", "err", err) return } hash := sha256.Sum256(b) if version == w.cur.version && hash == w.cur.fileHash { return } cluster := w.cur.cluster eff := file if w.st != nil { if version != w.cur.version { doc, err := w.st.ClusterSettings(ctx) if err != nil { return } if cluster, err = config.Parse([]byte(doc.Body)); err != nil { w.log.Error("the cluster's settings do not parse; running on what was in force", "version", doc.Version, "err", err) w.cur.version, w.cur.fileHash = version, hash return } } eff = config.Layer(file, cluster) } cfg, err := config.FromMap(eff) if err != nil { w.log.Error("the changed settings do not load on this machine; running on what was in force", "version", version, "err", err) w.cur.version, w.cur.fileHash = version, hash return } changed := config.Changed(w.cur.eff, eff) w.cur = clusterSettings{cfg: cfg, eff: eff, cluster: cluster, version: version, fileHash: hash} if len(changed) == 0 { return } if w.live != nil && w.live(cfg, changed) { w.log.Info("settings changed and in force", "version", version, "changed", strings.Join(changed, ", ")) return } w.restart(fmt.Sprintf("settings version %d changed %s", version, strings.Join(changed, ", "))) } // liveSettings are the keys a control plane puts in force without starting // again: what it reads at every use, or not at all. var liveSettings = []string{ "guard", // Stopping and starting the service (llmgw stop, llmgw start). "serving", // Sent to the nodes and this machine's firewall unit as they change. "firewall", "firewall_networks", // Sent to the nodes' agents as they change. "agents", // Read by the leader's presets pass each time it runs. "presets", // Read by server init, node add and backups, not by the server. "backup_recipients", "backup_copy_to", } // onlyLive is whether every changed key is one of liveSettings. func onlyLive(changed []string) bool { for _, c := range changed { top, _, _ := strings.Cut(c, ".") if !slices.Contains(liveSettings, top) { return false } } return true } // checkClusterSettings refuses a settings document that would not load, // laid over the server.yaml at path, or that sets one machine's keys. func checkClusterSettings(path string, body []byte) error { cluster, err := config.Parse(body) if err != nil { return err } if err := config.CheckCluster(cluster); err != nil { return err } _, file, err := readSettingsFile(path) if err != nil { return err } _, err = config.FromMap(config.Layer(file, cluster)) return err } // loadEffectiveConfig is what the control plane on this machine runs on: // its server.yaml with the cluster's settings from the store laid over it, // for the commands beside it that read cluster keys — init, node add, // backups. The file alone when the store cannot be reached or holds none. func loadEffectiveConfig(path string) (config.Server, error) { _, file, err := readSettingsFile(path) if err != nil { return config.Default(), err } fileCfg, err := config.FromMap(file) if err != nil { return fileCfg, fileError(path, err) } cluster, ok := storedSettings(fileCfg) if !ok { return fileCfg, nil } cfg, err := config.FromMap(config.Layer(file, cluster)) if err != nil { return fileCfg, nil //nolint:nilerr // the file still says what this machine is } return cfg, nil } // storedSettings are the cluster's settings, when the store a configuration // names can be reached and holds some. func storedSettings(cfg config.Server) (map[string]any, bool) { if cfg.Auth.Store == "" { return nil, false } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() st, err := storepkg.Open(ctx, cfg.Auth.Store) if err != nil { return nil, false } defer st.Close() doc, err := st.ClusterSettings(ctx) if err != nil || doc.Version == 0 { return nil, false } cluster, err := config.Parse([]byte(doc.Body)) return cluster, err == nil } // changeStoredSettings changes the cluster's settings from a command run on // a control plane (init), when the store holds them: changed is false when // it does not, and the file is then where the change belongs. func changeStoredSettings(path, note string, change func(doc *settings.Doc) error) (changed bool, err error) { _, file, err := readSettingsFile(path) if err != nil { return false, err } fileCfg, err := config.FromMap(file) if err != nil || fileCfg.Auth.Store == "" { return false, nil //nolint:nilerr // no store to keep them in } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() st, err := storepkg.Open(ctx, fileCfg.Auth.Store) if err != nil { return false, nil //nolint:nilerr // not set up yet: the file it is } defer st.Close() cur, err := st.ClusterSettings(ctx) if err != nil || cur.Version == 0 { return false, err } doc, err := settings.FromBytes("the cluster's settings", []byte(cur.Body)) if err != nil { return false, err } if err := change(doc); err != nil { return false, err } body, err := doc.Bytes() if err != nil { return false, err } if err := checkClusterSettings(path, body); err != nil { return false, err } who := "llmgw" if u := os.Getenv("SUDO_USER"); u != "" { who = u } _, err = st.PutClusterSettings(ctx, cur.Version, string(body), who, note) return err == nil, err }