Two bugs found while verifying mTLS against real machines. session cancelled its own derived context and then asked that context whether the agent was shutting down. The answer was always yes, so every stream error read as a clean close: the log said err=<nil>, the backoff reset, and an agent whose certificate the control plane refused hammered it forever at the base interval with nothing to diagnose. It now keeps the parent context to answer that question, and a refusal reaches the operator naming both the certificate and the claim. Cancelling the context also did not stop the agent. HTTP/2 watches the context while the request body is being written, and a bidirectional stream spends its life blocked on the read side instead, so the read never returned. Measured on a deployed node: `systemctl stop llmgw-agent` took 90 seconds and ended in SIGKILL, on every restart. Closing the response when the context ends is what actually unblocks it. TestAgentStopsWhenContextEnds hangs reliably without the second fix, which is the only reason to trust that it tests it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1e3cafdc86
commit
ec1b6b04df
9 files changed
+515
-27
No files matched your search
+92
-16
@@ -11,6 +11,7 @@ import (
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -23,6 +24,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
// Config configures the agent.
|
||||
@@ -45,6 +47,10 @@ type Config struct {
|
||||
ReconnectBase time.Duration `koanf:"reconnect_base"`
|
||||
ReconnectMax time.Duration `koanf:"reconnect_max"`
|
||||
|
||||
// TLS authenticates both ends of the control-plane stream. Leaving it unset
|
||||
// keeps the plaintext behaviour, which only belongs on a trusted network.
|
||||
TLS tlsconf.Config `koanf:"tls"`
|
||||
|
||||
Discovery discovery.Options `koanf:"discovery"`
|
||||
Engines engines.Config `koanf:"engines"`
|
||||
Superv supervisor.Config `koanf:"supervisor"`
|
||||
@@ -73,6 +79,12 @@ func (c *Config) normalize() error {
|
||||
if c.Server == "" {
|
||||
return errors.New("server is required")
|
||||
}
|
||||
if err := c.TLS.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.normalizeServerURL(); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.MgmtInterface == "" {
|
||||
c.MgmtInterface = d.MgmtInterface
|
||||
}
|
||||
@@ -88,6 +100,31 @@ func (c *Config) normalize() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeServerURL makes the configured address a URL connect can use, and
|
||||
// refuses the combination that would quietly undo the certificates: a scheme
|
||||
// that says plaintext while TLS is configured. Silently connecting anyway would
|
||||
// mean a cluster believing it has mutual authentication when it has none.
|
||||
func (c *Config) normalizeServerURL() error {
|
||||
secure := c.TLS.Enabled()
|
||||
switch {
|
||||
case strings.HasPrefix(c.Server, "https://"):
|
||||
if !secure {
|
||||
return errors.New("server is an https:// address but no tls certificates are configured")
|
||||
}
|
||||
case strings.HasPrefix(c.Server, "http://"):
|
||||
if secure {
|
||||
return errors.New("tls is configured but server is an http:// address; use https://")
|
||||
}
|
||||
case strings.Contains(c.Server, "://"):
|
||||
return fmt.Errorf("server %q must be an http:// or https:// address", c.Server)
|
||||
case secure:
|
||||
c.Server = "https://" + c.Server
|
||||
default:
|
||||
c.Server = "http://" + c.Server
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Agent runs on a GPU node.
|
||||
type Agent struct {
|
||||
cfg Config
|
||||
@@ -286,6 +323,10 @@ func (a *Agent) scan(ctx context.Context) error {
|
||||
func (a *Agent) session(ctx context.Context) error {
|
||||
client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server)
|
||||
|
||||
// parent is kept because the derived context is cancelled from inside this
|
||||
// function; asking the derived one whether we are shutting down would
|
||||
// always say yes, and every stream error would read as a clean close.
|
||||
parent := ctx
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
@@ -310,6 +351,22 @@ func (a *Agent) session(ctx context.Context) error {
|
||||
return fmt.Errorf("registering: %w", err)
|
||||
}
|
||||
|
||||
// Closing the response is what actually unblocks Receive below. Cancelling
|
||||
// the context is not enough: HTTP/2 watches it while the request body is
|
||||
// being written, and a bidirectional stream spends its life blocked on the
|
||||
// read side instead. Without this an agent told to stop keeps reading until
|
||||
// something kills it — which for systemd means the full stop timeout on
|
||||
// every restart.
|
||||
watchDone := make(chan struct{})
|
||||
defer close(watchDone)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = stream.CloseResponse()
|
||||
case <-watchDone:
|
||||
}
|
||||
}()
|
||||
|
||||
// Sends are serialized through this channel: connect streams are not safe
|
||||
// for concurrent writes, and heartbeats race with status reports.
|
||||
out := make(chan *llmgwv1.AgentMessage, 32)
|
||||
@@ -325,9 +382,18 @@ func (a *Agent) session(ctx context.Context) error {
|
||||
msg, err := stream.Receive()
|
||||
if err != nil {
|
||||
cancel()
|
||||
if errors.Is(err, io.EOF) || ctx.Err() != nil {
|
||||
switch {
|
||||
case parent.Err() != nil:
|
||||
// The agent is stopping; the stream ending is the consequence.
|
||||
return nil
|
||||
case errors.Is(err, io.EOF):
|
||||
// The control plane closed the stream, which a restart does.
|
||||
// Reconnect immediately rather than backing off.
|
||||
return nil
|
||||
}
|
||||
// Everything else is worth both reporting and backing off from. A
|
||||
// refused certificate arrives here, and an agent that cannot say
|
||||
// why it is not registering is one an operator cannot fix.
|
||||
return err
|
||||
}
|
||||
a.handle(ctx, msg, out)
|
||||
@@ -536,23 +602,33 @@ func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo {
|
||||
return out
|
||||
}
|
||||
|
||||
// HTTP2Client returns the client the agent should use to reach the control
|
||||
// plane over plaintext HTTP/2.
|
||||
// Client returns the client the agent reaches the control plane with.
|
||||
//
|
||||
// connect's bidirectional streams require HTTP/2, and an agent talking to a
|
||||
// control plane over a trusted network has no TLS to negotiate it with. Go's
|
||||
// default transport will not speak HTTP/2 without TLS, so it has to be asked
|
||||
// explicitly.
|
||||
// Both branches speak HTTP/2, which connect's bidirectional streams require and
|
||||
// which Go will not do by default in either case: without TLS it has to be
|
||||
// asked explicitly, and with TLS it has to be negotiated through ALPN.
|
||||
//
|
||||
// mTLS is plan §5.6's target and replaces this; until then the stream must not
|
||||
// cross an untrusted network.
|
||||
func HTTP2Client(timeout time.Duration) *http.Client {
|
||||
return &http.Client{
|
||||
Transport: &http2.Transport{
|
||||
AllowHTTP: true,
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
|
||||
// The plaintext branch exists for local runs against sim-engines. Anything
|
||||
// using it must stay on a trusted network, because nothing there stops a node
|
||||
// from claiming to be another one.
|
||||
func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
|
||||
if !cfg.Enabled() {
|
||||
return &http.Client{
|
||||
Transport: &http2.Transport{
|
||||
AllowHTTP: true,
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ServerName is left to the transport, which takes it from the address
|
||||
// being dialled. Hardcoding it here would break the moment the control
|
||||
// plane moved.
|
||||
tc, err := cfg.ClientConfig("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
// pki issues a cluster's certificates into a temporary directory and returns
|
||||
// the configurations the control plane and one agent would load.
|
||||
func pki(t *testing.T, nodeID string) (server, client tlsconf.Config) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
|
||||
ca, err := tlsconf.NewCA("test CA")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ca.Write(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv, err := ca.IssueServer("control-plane", []string{"127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := srv.Write(dir, "server"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cli, err := ca.IssueClient(nodeID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cli.Write(dir, "agent"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
caPath := filepath.Join(dir, "ca.pem")
|
||||
return tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")},
|
||||
tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "agent.pem"), Key: filepath.Join(dir, "agent-key.pem")}
|
||||
}
|
||||
|
||||
// startTLS runs a real service behind a real TLS listener, with identity
|
||||
// required — the deployed configuration rather than an approximation of it.
|
||||
func startTLS(t *testing.T, serverTLS tlsconf.Config) (*agentsvc.Service, *inventory.Store, string) {
|
||||
t.Helper()
|
||||
|
||||
inv := inventory.NewStore()
|
||||
svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler))
|
||||
path, handler := svc.Handler()
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(path, handler)
|
||||
|
||||
tc, err := serverTLS.ServerConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewUnstartedServer(mux)
|
||||
srv.Listener = raw
|
||||
srv.TLS = tc
|
||||
srv.StartTLS()
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
return svc, inv, "https://" + raw.Addr().String()
|
||||
}
|
||||
|
||||
func runAgent(t *testing.T, cfg Config) *Agent {
|
||||
t.Helper()
|
||||
|
||||
cfg.MgmtAddr = "127.0.0.1"
|
||||
cfg.HeartbeatInterval = 50 * time.Millisecond
|
||||
cfg.ReconnectBase = 20 * time.Millisecond
|
||||
cfg.ReconnectMax = 100 * time.Millisecond
|
||||
cfg.Discovery.SkipGPUs = true
|
||||
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||
cfg.Superv = supervisor.Config{
|
||||
WorkDir: t.TempDir(), LogLines: 64,
|
||||
StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := Client(cfg.TLS, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() { _ = ag.Run(ctx) }()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
ag.Supervisor().StopAll(context.Background())
|
||||
})
|
||||
return ag
|
||||
}
|
||||
|
||||
func registered(svc *agentsvc.Service, inv *inventory.Store, nodeID string) func() bool {
|
||||
return func() bool {
|
||||
_, ok := inv.Get(nodeID)
|
||||
return ok && len(svc.ConnectedNodes()) == 1
|
||||
}
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, limit time.Duration, pred func() bool) bool {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(limit)
|
||||
for time.Now().Before(deadline) {
|
||||
if pred() {
|
||||
return true
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestAgentRegistersOverMutualTLS(t *testing.T) {
|
||||
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||
svc, inv, url := startTLS(t, serverTLS)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node01"
|
||||
cfg.Server = url
|
||||
cfg.TLS = clientTLS
|
||||
runAgent(t, cfg)
|
||||
|
||||
if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) {
|
||||
t.Fatal("the agent never registered over mutual TLS")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeCannotClaimAnotherNodesIdentity(t *testing.T) {
|
||||
// The point of the whole exercise. node01 holds a valid certificate — it is
|
||||
// a legitimate member of the cluster — and still cannot register as node02,
|
||||
// because the id it claims is checked against the one it can prove.
|
||||
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||
svc, inv, url := startTLS(t, serverTLS)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node02"
|
||||
cfg.Server = url
|
||||
cfg.TLS = clientTLS
|
||||
runAgent(t, cfg)
|
||||
|
||||
// The agent reconnects on failure, so this has to stay false throughout
|
||||
// rather than merely start false.
|
||||
if eventually(t, 2*time.Second, func() bool {
|
||||
_, ok := inv.Get("llmgw-node02")
|
||||
return ok || len(svc.ConnectedNodes()) > 0
|
||||
}) {
|
||||
t.Fatal("an agent registered under a node id its certificate does not authorize")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnauthenticatedAgentIsRefused(t *testing.T) {
|
||||
// The defensive half of the check: identity is required, but the connection
|
||||
// carried none. In the shipped wiring TLS and this flag are tied together,
|
||||
// so reaching here means someone untied them — and the answer must still be
|
||||
// refusal rather than a silently trusted node id.
|
||||
inv := inventory.NewStore()
|
||||
svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler))
|
||||
path, handler := svc.Handler()
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(path, handler)
|
||||
|
||||
srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node01"
|
||||
cfg.Server = srv.URL
|
||||
runAgent(t, cfg)
|
||||
|
||||
if eventually(t, 2*time.Second, registered(svc, inv, "llmgw-node01")) {
|
||||
t.Fatal("an agent with no proven identity reached the control plane")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlaintextAgentIsRejectedWhenTLSIsConfigured(t *testing.T) {
|
||||
// A misconfiguration worth catching at startup rather than at runtime: an
|
||||
// agent holding certificates but dialling http:// would connect in the
|
||||
// clear and never learn that it had.
|
||||
_, clientTLS := pki(t, "llmgw-node01")
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node01"
|
||||
cfg.Server = "http://127.0.0.1:9443"
|
||||
cfg.TLS = clientTLS
|
||||
|
||||
if _, err := New(cfg, nil, slog.New(slog.DiscardHandler)); err == nil {
|
||||
t.Fatal("an agent with certificates was allowed to dial a plaintext address")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBareAddressGainsTheRightScheme(t *testing.T) {
|
||||
_, clientTLS := pki(t, "llmgw-node01")
|
||||
|
||||
secure := Config{NodeID: "n", Server: "10.0.0.140:9443", TLS: clientTLS}
|
||||
if err := secure.normalize(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if secure.Server != "https://10.0.0.140:9443" {
|
||||
t.Errorf("a configured agent dials %q", secure.Server)
|
||||
}
|
||||
|
||||
plain := Config{NodeID: "n", Server: "10.0.0.140:9443"}
|
||||
if err := plain.normalize(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plain.Server != "http://10.0.0.140:9443" {
|
||||
t.Errorf("an unconfigured agent dials %q", plain.Server)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefusedAgentLearnsWhy(t *testing.T) {
|
||||
// An agent that cannot register must be able to say why. Reporting a
|
||||
// refusal as a clean disconnect leaves an operator with a silent reconnect
|
||||
// loop and nothing to act on — and resets the backoff, so the loop is a
|
||||
// fast one.
|
||||
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||
_, _, url := startTLS(t, serverTLS)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node02"
|
||||
cfg.Server = url
|
||||
cfg.TLS = clientTLS
|
||||
cfg.Discovery.SkipGPUs = true
|
||||
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||
cfg.Superv = supervisor.Config{WorkDir: t.TempDir(), LogLines: 8}
|
||||
|
||||
client, err := Client(cfg.TLS, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ag.session(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("a refused session reported success")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "llmgw-node01") || !strings.Contains(err.Error(), "llmgw-node02") {
|
||||
t.Errorf("the error names neither certificate nor claim: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
)
|
||||
|
||||
// TestAgentStopsWhenContextEnds pins down something cancellation does not do by
|
||||
// itself. A bidirectional stream spends its life blocked reading, and HTTP/2
|
||||
// stops watching the context once the request headers are away, so an agent
|
||||
// that only cancels keeps reading until something kills it. Under systemd that
|
||||
// is the full stop timeout on every restart, followed by SIGKILL.
|
||||
func TestAgentStopsWhenContextEnds(t *testing.T) {
|
||||
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||
svc, inv, url := startTLS(t, serverTLS)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node01"
|
||||
cfg.Server = url
|
||||
cfg.TLS = clientTLS
|
||||
cfg.HeartbeatInterval = 50 * time.Millisecond
|
||||
cfg.Discovery.SkipGPUs = true
|
||||
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||
cfg.Superv = supervisor.Config{
|
||||
WorkDir: t.TempDir(), LogLines: 8,
|
||||
StopTimeout: time.Second, HealthInterval: 20 * time.Millisecond,
|
||||
}
|
||||
|
||||
client, err := Client(cfg.TLS, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { _ = ag.Run(ctx); close(done) }()
|
||||
|
||||
// Cancelling before the stream is established would prove nothing: the
|
||||
// blocked read is the case that matters.
|
||||
if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) {
|
||||
t.Fatal("the agent never registered")
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the agent did not stop within 5s of being told to")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user