Ilker ManapandClaude Opus 5
4ba84fa90e
The second half of leadership. A standby that refuses to place but still holds an agent's stream is worse than useless: commands travel down the stream the agent opened, so the leader's placements fail with "no agent connected" for a node that is plainly connected. So a standby refuses the stream, before reading the opening message, and says why. And an agent takes a list rather than an address, trying each in turn — a standby refusing, a stopped control plane refusing the connection and a crashed one never answering are the same thing from a node, and the answer to all three is somewhere else. `server` and `servers` are both accepted, with `server` first when both are given: every agent configuration written before there could be a second control plane names one address, and an operator adding a standby expects the one already working to stay preferred. With a single address the behaviour is exactly what it was, including the backoff — which is skipped only while there is another address that has not just been tried, because the delay exists to stop hammering one control plane and the next one has not been hammered. Removed a check that asked whether one field was empty, which is how the first run of these tests failed: the list was populated and `server` was not, and an agent configured entirely correctly was refused by its own validation. The tests run a real agent against a real service over h2c. The first attempt used a plain HTTP server, and the refusal arrived as an HTTP/2 framing error that said nothing about why — which is exactly what an operator would have seen. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
108 lines
3.3 KiB
Go
108 lines
3.3 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/net/http2"
|
|
"golang.org/x/net/http2/h2c"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
type standby bool
|
|
|
|
func (s standby) IsLeader() bool { return !bool(s) }
|
|
|
|
// startPlain runs an agent service over h2c, which is enough to test who
|
|
// accepts a connection and who refuses it.
|
|
//
|
|
// A nil leader means none was configured, which is every deployment that
|
|
// existed before leadership did and has to keep accepting agents.
|
|
func startPlain(t *testing.T, leader agentsvc.Leader) (*inventory.Store, string) {
|
|
t.Helper()
|
|
inv := inventory.NewStore()
|
|
svc := agentsvc.New(inv, newRecorder(), false, slog.New(slog.DiscardHandler))
|
|
if leader != nil {
|
|
svc.SetLeadership(leader)
|
|
}
|
|
|
|
path, handler := svc.Handler()
|
|
mux := http.NewServeMux()
|
|
mux.Handle(path, handler)
|
|
// h2c, because the agent speaks HTTP/2 with prior knowledge and a plain
|
|
// server negotiates HTTP/1.1 — which turns a clear refusal into a frame
|
|
// parsing error that says nothing about why.
|
|
srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
|
|
t.Cleanup(srv.Close)
|
|
return inv, srv.URL
|
|
}
|
|
|
|
func TestAStandbyRefusesAnAgent(t *testing.T) {
|
|
// Commands travel down the stream the agent opened, so whichever control
|
|
// plane holds it is the only one that can start anything on that node. A
|
|
// standby holding it means the leader's placements fail with "no agent
|
|
// connected" for a node that is plainly connected.
|
|
_, addr := startPlain(t, standby(true))
|
|
|
|
ag := runAgent(t, Config{NodeID: "n1", Server: addr, MgmtAddr: "127.0.0.1"})
|
|
err := ag.session(context.Background(), ag.cfg.Server)
|
|
if err == nil {
|
|
t.Fatal("a standby accepted an agent")
|
|
}
|
|
if !strings.Contains(err.Error(), "standby") {
|
|
t.Errorf("the refusal does not say why: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAnAgentReachesTheLeaderPastAStandby(t *testing.T) {
|
|
// The point of the list. The standby is first, so the agent only registers
|
|
// if it moved on rather than sitting on the refusal.
|
|
_, standbyAddr := startPlain(t, standby(true))
|
|
leaderInv, leaderAddr := startPlain(t, standby(false))
|
|
|
|
ag := runAgent(t, Config{
|
|
NodeID: "n1", MgmtAddr: "127.0.0.1",
|
|
Servers: []string{standbyAddr, leaderAddr},
|
|
})
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
go ag.Run(ctx)
|
|
|
|
deadline := time.Now().Add(8 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if _, ok := leaderInv.Get("n1"); ok {
|
|
return
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
t.Fatal("the agent never reached the leader past the standby")
|
|
}
|
|
|
|
func TestALeaderStillAcceptsWithNoLeadershipConfigured(t *testing.T) {
|
|
// Every deployment before this had no leadership at all, and must keep
|
|
// accepting agents with nothing configured.
|
|
inv, addr := startPlain(t, nil)
|
|
|
|
ag := runAgent(t, Config{NodeID: "n1", Server: addr, MgmtAddr: "127.0.0.1"})
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
go ag.Run(ctx)
|
|
|
|
deadline := time.Now().Add(4 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if _, ok := inv.Get("n1"); ok {
|
|
return
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
t.Fatal("a service with no leadership configured refused an agent")
|
|
}
|