The nodes' agent settings kept with the cluster's, agent.yaml down to what an agent needs to start
agents.default and agents.nodes.<id> in the cluster's settings hold everything agent.yaml can say but the agent's id, control planes, address and certificate, which stay in the file and are refused there. The control plane sends each node its settings on register and, a second apart, when they change; an agent whose settings changed checks they load beside its file, keeps them and starts again on them, engines serving throughout. An edited agent.yaml that loads restarts the agent too. History, audit, ctl config, the console and the portal's settings page cover them as the rest of the settings. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
659f20dc9a
commit
dc243f8840
21 files changed
+917
-229
No files matched your search
@@ -0,0 +1,50 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
// SetAgentSettings gives the service each node's agent settings from the
|
||||
// cluster's (BACKLOG E3), as YAML, read at each use so a change is what is
|
||||
// sent next.
|
||||
func (s *Service) SetAgentSettings(f func(node string) string) {
|
||||
s.mu.Lock()
|
||||
s.agentSettings = f
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Service) agentSettingsMsg(node string) *llmgwv1.ControlMessage {
|
||||
s.mu.RLock()
|
||||
f := s.agentSettings
|
||||
s.mu.RUnlock()
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
return &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_AgentSettings{
|
||||
AgentSettings: &llmgwv1.AgentSettings{Yaml: f(node)}}}
|
||||
}
|
||||
|
||||
// agentSettingsGap is the pause between telling one node and the next: an
|
||||
// agent whose settings changed starts again, and a whole cluster's agents
|
||||
// doing so in the same second would all re-register at once.
|
||||
var agentSettingsGap = time.Second
|
||||
|
||||
// BroadcastAgentSettings tells every connected agent what the cluster's
|
||||
// settings now say it runs on, a moment apart. A node not connected is told
|
||||
// when it registers.
|
||||
func (s *Service) BroadcastAgentSettings() {
|
||||
for _, node := range s.ConnectedNodes() {
|
||||
msg := s.agentSettingsMsg(node)
|
||||
if msg == nil {
|
||||
return
|
||||
}
|
||||
if err := s.send(node, msg); err != nil {
|
||||
s.log.Warn("could not tell a node its agent settings; it is told when it next registers",
|
||||
"node", node, "err", err)
|
||||
continue
|
||||
}
|
||||
time.Sleep(agentSettingsGap)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func TestEachNodeIsToldItsOwnAgentSettings(t *testing.T) {
|
||||
s := New(inventory.NewStore(), nil, false, slog.New(slog.DiscardHandler))
|
||||
if s.agentSettingsMsg("n1") != nil {
|
||||
t.Error("told settings with nothing to read them from")
|
||||
}
|
||||
s.SetAgentSettings(func(node string) string {
|
||||
if node == "cuda01" {
|
||||
return "labels:\n gpu: rtx6000\n"
|
||||
}
|
||||
return ""
|
||||
})
|
||||
if got := s.agentSettingsMsg("cuda01").GetAgentSettings().GetYaml(); got != "labels:\n gpu: rtx6000\n" {
|
||||
t.Errorf("cuda01 told %q", got)
|
||||
}
|
||||
// None is said too, so an agent whose settings were removed drops them.
|
||||
if m := s.agentSettingsMsg("node01"); m == nil || m.GetAgentSettings().GetYaml() != "" {
|
||||
t.Errorf("node01 told %v", m)
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,9 @@ type Service struct {
|
||||
controlPlanes func(context.Context) ([]string, error)
|
||||
// firewall is what nodes are told of llmgw's firewall; see SetFirewall.
|
||||
firewall func() *llmgwv1.Firewall
|
||||
// agentSettings is what each node's agent is told to run on; see
|
||||
// SetAgentSettings.
|
||||
agentSettings func(node string) string
|
||||
|
||||
// logWaits are the log requests awaiting a reply, by request id.
|
||||
logMu sync.Mutex
|
||||
@@ -296,6 +299,13 @@ func (s *Service) Connect(
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What the cluster's settings say this agent runs on. An agent already
|
||||
// on them ignores it; one that is not starts again on them.
|
||||
if msg := s.agentSettingsMsg(nodeID); msg != nil {
|
||||
if err := stream.Send(msg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Recover replica state from what this node is already running, before any
|
||||
// reconciliation pass can conclude the cluster is empty and place
|
||||
|
||||
Reference in new issue
Block a user