The nodes' agent settings kept with the cluster's, agent.yaml down to what an agent needs to start
agents.default and agents.nodes.<id> in the cluster's settings hold everything agent.yaml can say but the agent's id, control planes, address and certificate, which stay in the file and are refused there. The control plane sends each node its settings on register and, a second apart, when they change; an agent whose settings changed checks they load beside its file, keeps them and starts again on them, engines serving throughout. An edited agent.yaml that loads restarts the agent too. History, audit, ctl config, the console and the portal's settings page cover them as the rest of the settings. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
659f20dc9a
commit
dc243f8840
21 files changed
+917
-229
No files matched your search
@@ -136,6 +136,10 @@ type Agent struct {
|
||||
// by mu.
|
||||
kv *kvevents.Mirror
|
||||
kvSubs map[string]*kvevents.Subscriber
|
||||
|
||||
// settingsCheck says whether the cluster's settings for this agent
|
||||
// would load beside its agent.yaml; see SetSettingsCheck.
|
||||
settingsCheck func([]byte) error
|
||||
}
|
||||
|
||||
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
|
||||
@@ -880,6 +884,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.setControlPlanes(m.ControlPlanes)
|
||||
case *llmgwv1.ControlMessage_Firewall:
|
||||
go a.applyFirewall(m.Firewall)
|
||||
case *llmgwv1.ControlMessage_AgentSettings:
|
||||
go a.applySettings(m.AgentSettings)
|
||||
case *llmgwv1.ControlMessage_RestartAgent:
|
||||
go func() {
|
||||
a.log.Info("restarting, as the control plane asks; the engines keep serving",
|
||||
|
||||
@@ -103,6 +103,11 @@ type Config struct {
|
||||
Tracing obs.Tracing `koanf:"tracing"`
|
||||
|
||||
Version string
|
||||
|
||||
// Settings is the cluster's settings for this agent it was started on,
|
||||
// as the control plane sent them: what a newly sent version is
|
||||
// compared with.
|
||||
Settings string `koanf:"-"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
// The cluster's settings for this agent (BACKLOG E3): what agent.yaml can
|
||||
// say but the agent's id, control planes, address and certificate, kept by
|
||||
// the control planes and sent over the stream. Kept here beside agent.yaml,
|
||||
// which then holds only what the agent needs to start; at start the two are
|
||||
// read together, these over the file.
|
||||
|
||||
// SettingsFile keeps the settings the control plane last sent.
|
||||
var SettingsFile = "/var/lib/llmgw-agent/agent-settings.yaml"
|
||||
|
||||
// ReadSettings is what SettingsFile holds, nil when nothing.
|
||||
func ReadSettings() []byte {
|
||||
b, err := os.ReadFile(SettingsFile)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// SetSettingsCheck says how to tell whether settings would load beside this
|
||||
// agent's own file; settings that would not are not taken.
|
||||
func (a *Agent) SetSettingsCheck(f func([]byte) error) { a.settingsCheck = f }
|
||||
|
||||
// applySettings takes the settings the control plane sent: nothing when they
|
||||
// are what this agent runs on, and otherwise kept and started again on.
|
||||
func (a *Agent) applySettings(m *llmgwv1.AgentSettings) {
|
||||
next := m.GetYaml()
|
||||
if next == a.cfg.Settings {
|
||||
return
|
||||
}
|
||||
if a.settingsCheck != nil {
|
||||
if err := a.settingsCheck([]byte(next)); err != nil {
|
||||
// Kept running on what it has: settings that will not load
|
||||
// here must not take the node out of service.
|
||||
a.log.Error("the cluster's settings for this agent do not load beside its agent.yaml; running on what it has",
|
||||
"err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(SettingsFile), 0o755); err != nil {
|
||||
a.log.Error("could not keep the cluster's settings for this agent", "err", err)
|
||||
return
|
||||
}
|
||||
tmp := SettingsFile + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(next), 0o600); err != nil {
|
||||
a.log.Error("could not keep the cluster's settings for this agent", "err", err)
|
||||
return
|
||||
}
|
||||
if err := os.Rename(tmp, SettingsFile); err != nil {
|
||||
a.log.Error("could not keep the cluster's settings for this agent", "err", err)
|
||||
return
|
||||
}
|
||||
a.log.Info("the cluster's settings for this agent changed; starting again on them, the engines keep serving")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}
|
||||
|
||||
// WatchFile starts the agent again when its configuration file changes and
|
||||
// still loads: agent.yaml is read only at start, and a change to it that
|
||||
// waited for somebody to restart the agent was a change nobody saw take
|
||||
// effect.
|
||||
func (a *Agent) WatchFile(ctx context.Context, path string, loads func() error) {
|
||||
if path == "" {
|
||||
return
|
||||
}
|
||||
sum := func() [32]byte {
|
||||
b, _ := os.ReadFile(path) //nolint:gosec // the agent's own configuration
|
||||
return sha256.Sum256(b)
|
||||
}
|
||||
was := sum()
|
||||
t := time.NewTicker(watchEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
now := sum()
|
||||
if now == was {
|
||||
continue
|
||||
}
|
||||
if err := loads(); err != nil {
|
||||
// Once per change: an editor saving halfway must not stop the
|
||||
// agent, and a broken file is reported, not restarted into.
|
||||
a.log.Error("agent.yaml changed and does not load; running on what it had", "path", path, "err", err)
|
||||
was = now
|
||||
continue
|
||||
}
|
||||
a.log.Info("agent.yaml changed; starting again on it, the engines keep serving", "path", path)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// watchEvery is how often agent.yaml is looked at.
|
||||
var watchEvery = 10 * time.Second
|
||||
@@ -0,0 +1,88 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
func settingsAgent(t *testing.T, running string) (*Agent, chan struct{}) {
|
||||
t.Helper()
|
||||
oldFile, oldExit := SettingsFile, exitForUpgrade
|
||||
SettingsFile = filepath.Join(t.TempDir(), "agent-settings.yaml")
|
||||
exited := make(chan struct{}, 1)
|
||||
exitForUpgrade = func() { exited <- struct{}{} }
|
||||
t.Cleanup(func() { SettingsFile, exitForUpgrade = oldFile, oldExit })
|
||||
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir(), Settings: running}
|
||||
cfg.Superv.WorkDir = t.TempDir()
|
||||
a, err := New(cfg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return a, exited
|
||||
}
|
||||
|
||||
// exitedSoon is whether the agent exited within a second: it waits a
|
||||
// moment for its log line to leave first.
|
||||
func exitedSoon(ch chan struct{}) bool {
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
case <-time.After(time.Second):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAgentOnItsSettingsStaysAndOneOffThemStartsAgain(t *testing.T) {
|
||||
a, exited := settingsAgent(t, "heartbeat_interval: 5s\n")
|
||||
a.applySettings(&llmgwv1.AgentSettings{Yaml: "heartbeat_interval: 5s\n"})
|
||||
if exitedSoon(exited) || ReadSettings() != nil {
|
||||
t.Error("settings it already runs on restarted it")
|
||||
}
|
||||
a.applySettings(&llmgwv1.AgentSettings{Yaml: "heartbeat_interval: 2s\n"})
|
||||
if !exitedSoon(exited) || string(ReadSettings()) != "heartbeat_interval: 2s\n" {
|
||||
t.Errorf("new settings: kept %q", ReadSettings())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsThatWouldNotLoadAreNotTaken(t *testing.T) {
|
||||
a, exited := settingsAgent(t, "")
|
||||
a.SetSettingsCheck(func([]byte) error { return errors.New("fabric ib-a lists no HCAs") })
|
||||
a.applySettings(&llmgwv1.AgentSettings{Yaml: "fabrics: [{name: ib-a}]\n"})
|
||||
if exitedSoon(exited) || ReadSettings() != nil {
|
||||
t.Error("settings that do not load were taken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAChangedAgentYamlThatLoadsStartsTheAgentAgain(t *testing.T) {
|
||||
a, exited := settingsAgent(t, "")
|
||||
old := watchEvery
|
||||
watchEvery = 10 * time.Millisecond
|
||||
t.Cleanup(func() { watchEvery = old })
|
||||
p := filepath.Join(t.TempDir(), "agent.yaml")
|
||||
_ = os.WriteFile(p, []byte("node_id: n1\n"), 0o600)
|
||||
var loads atomic.Bool
|
||||
check := func() error {
|
||||
if loads.Load() {
|
||||
return nil
|
||||
}
|
||||
return errors.New("broken")
|
||||
}
|
||||
go a.WatchFile(t.Context(), p, check)
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
// Saved broken: reported, not restarted into.
|
||||
_ = os.WriteFile(p, []byte("node_id: [\n"), 0o600)
|
||||
if exitedSoon(exited) {
|
||||
t.Fatal("a broken agent.yaml restarted the agent")
|
||||
}
|
||||
loads.Store(true)
|
||||
_ = os.WriteFile(p, []byte("node_id: n1\nlabels: {a: b}\n"), 0o600)
|
||||
if !exitedSoon(exited) {
|
||||
t.Error("a changed agent.yaml did not restart the agent")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user