From ec1b6b04df1fb4d2b4f0677c47d54e391332f74f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 08:03:21 +0200 Subject: [PATCH] Fix an agent that could neither explain itself nor stop Two bugs found while verifying mTLS against real machines. session cancelled its own derived context and then asked that context whether the agent was shutting down. The answer was always yes, so every stream error read as a clean close: the log said err=, the backoff reset, and an agent whose certificate the control plane refused hammered it forever at the base interval with nothing to diagnose. It now keeps the parent context to answer that question, and a refusal reaches the operator naming both the certificate and the claim. Cancelling the context also did not stop the agent. HTTP/2 watches the context while the request body is being written, and a bidirectional stream spends its life blocked on the read side instead, so the read never returned. Measured on a deployed node: `systemctl stop llmgw-agent` took 90 seconds and ended in SIGKILL, on every restart. Closing the response when the context ends is what actually unblocks it. TestAgentStopsWhenContextEnds hangs reliably without the second fix, which is the only reason to trust that it tests it. Co-Authored-By: Claude Opus 5 (1M context) --- deploy/examples/agent.yaml | 15 ++ deploy/examples/server.yaml | 27 +++- internal/agent/agent.go | 108 +++++++++++-- internal/agent/mtls_test.go | 265 ++++++++++++++++++++++++++++++++ internal/agent/shutdown_test.go | 59 +++++++ internal/cli/agent.go | 7 +- internal/cli/server.go | 31 +++- internal/config/config.go | 9 ++ llmgw-plan.md | 21 ++- 9 files changed, 515 insertions(+), 27 deletions(-) create mode 100644 internal/agent/mtls_test.go create mode 100644 internal/agent/shutdown_test.go diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml index 50ed81e..a78f521 100644 --- a/deploy/examples/agent.yaml +++ b/deploy/examples/agent.yaml @@ -7,6 +7,21 @@ node_id: gpu-a-01 # defaults to the hostname server: "http://llmgw-server.internal:7443" +# The agent's half of mutual TLS. The certificate's common name must be this +# node's id, because the control plane compares the two and refuses the stream +# when they disagree: +# +# llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki +# +# With this set, `server` above must be an https:// address — an agent holding +# certificates and dialling http:// is refused at startup rather than connecting +# in the clear without noticing. +# +# tls: +# ca: /etc/llmgw/ca.pem +# cert: /etc/llmgw/agent.pem +# key: /etc/llmgw/agent-key.pem + # The address engines are reachable on, and the address NCCL bootstraps over. # Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA. mgmt_addr: "10.0.0.11" diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 439587e..e2668db 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -9,10 +9,33 @@ listen: ":8080" # client-facing OpenAI API admin_listen: ":9090" # metrics, health, pprof — do not expose this publicly # Where node agents connect. Separate from the client port so the fleet-facing -# surface can be firewalled independently. Agents speak plaintext HTTP/2 today, -# so this must stay on a trusted network until mTLS lands (plan §5.6). +# surface can be firewalled independently. agent_listen: ":7443" +# Mutual TLS on the agent port (plan §5.6). With it, a node's identity is taken +# from its certificate's common name and an agent claiming a different node id +# is refused. Without it the port is plaintext HTTP/2 and anyone who can reach +# it can claim to be any node, so it must stay on a trusted network. +# +# These certificates come from llmgw's own authority, not a public one: public +# authorities stopped issuing the clientAuth usage, so no Let's Encrypt +# certificate can authenticate an agent. +# +# llmgw ctl cert init-ca --dir /etc/llmgw/pki +# llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.140 --dir /etc/llmgw/pki +# +# Every address an agent might dial has to appear in --host, bare IP addresses +# included: hostname verification reads subject alternative names and ignores +# the common name entirely. +# +# This covers the fleet-facing port only. The client-facing API above is public +# and belongs behind a publicly trusted certificate. +# +# agent_tls: +# ca: /etc/llmgw/ca.pem +# cert: /etc/llmgw/server.pem +# key: /etc/llmgw/server-key.pem + router: scrape_interval: 2s scrape_timeout: 1s diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 2b671bd..d21e0f7 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -11,6 +11,7 @@ import ( "math/rand/v2" "net" "net/http" + "strings" "sync" "time" @@ -23,6 +24,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Config configures the agent. @@ -45,6 +47,10 @@ type Config struct { ReconnectBase time.Duration `koanf:"reconnect_base"` ReconnectMax time.Duration `koanf:"reconnect_max"` + // TLS authenticates both ends of the control-plane stream. Leaving it unset + // keeps the plaintext behaviour, which only belongs on a trusted network. + TLS tlsconf.Config `koanf:"tls"` + Discovery discovery.Options `koanf:"discovery"` Engines engines.Config `koanf:"engines"` Superv supervisor.Config `koanf:"supervisor"` @@ -73,6 +79,12 @@ func (c *Config) normalize() error { if c.Server == "" { return errors.New("server is required") } + if err := c.TLS.Validate(); err != nil { + return err + } + if err := c.normalizeServerURL(); err != nil { + return err + } if c.MgmtInterface == "" { c.MgmtInterface = d.MgmtInterface } @@ -88,6 +100,31 @@ func (c *Config) normalize() error { return nil } +// normalizeServerURL makes the configured address a URL connect can use, and +// refuses the combination that would quietly undo the certificates: a scheme +// that says plaintext while TLS is configured. Silently connecting anyway would +// mean a cluster believing it has mutual authentication when it has none. +func (c *Config) normalizeServerURL() error { + secure := c.TLS.Enabled() + switch { + case strings.HasPrefix(c.Server, "https://"): + if !secure { + return errors.New("server is an https:// address but no tls certificates are configured") + } + case strings.HasPrefix(c.Server, "http://"): + if secure { + return errors.New("tls is configured but server is an http:// address; use https://") + } + case strings.Contains(c.Server, "://"): + return fmt.Errorf("server %q must be an http:// or https:// address", c.Server) + case secure: + c.Server = "https://" + c.Server + default: + c.Server = "http://" + c.Server + } + return nil +} + // Agent runs on a GPU node. type Agent struct { cfg Config @@ -286,6 +323,10 @@ func (a *Agent) scan(ctx context.Context) error { func (a *Agent) session(ctx context.Context) error { client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server) + // parent is kept because the derived context is cancelled from inside this + // function; asking the derived one whether we are shutting down would + // always say yes, and every stream error would read as a clean close. + parent := ctx ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -310,6 +351,22 @@ func (a *Agent) session(ctx context.Context) error { return fmt.Errorf("registering: %w", err) } + // Closing the response is what actually unblocks Receive below. Cancelling + // the context is not enough: HTTP/2 watches it while the request body is + // being written, and a bidirectional stream spends its life blocked on the + // read side instead. Without this an agent told to stop keeps reading until + // something kills it — which for systemd means the full stop timeout on + // every restart. + watchDone := make(chan struct{}) + defer close(watchDone) + go func() { + select { + case <-ctx.Done(): + _ = stream.CloseResponse() + case <-watchDone: + } + }() + // Sends are serialized through this channel: connect streams are not safe // for concurrent writes, and heartbeats race with status reports. out := make(chan *llmgwv1.AgentMessage, 32) @@ -325,9 +382,18 @@ func (a *Agent) session(ctx context.Context) error { msg, err := stream.Receive() if err != nil { cancel() - if errors.Is(err, io.EOF) || ctx.Err() != nil { + switch { + case parent.Err() != nil: + // The agent is stopping; the stream ending is the consequence. + return nil + case errors.Is(err, io.EOF): + // The control plane closed the stream, which a restart does. + // Reconnect immediately rather than backing off. return nil } + // Everything else is worth both reporting and backing off from. A + // refused certificate arrives here, and an agent that cannot say + // why it is not registering is one an operator cannot fix. return err } a.handle(ctx, msg, out) @@ -536,23 +602,33 @@ func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo { return out } -// HTTP2Client returns the client the agent should use to reach the control -// plane over plaintext HTTP/2. +// Client returns the client the agent reaches the control plane with. // -// connect's bidirectional streams require HTTP/2, and an agent talking to a -// control plane over a trusted network has no TLS to negotiate it with. Go's -// default transport will not speak HTTP/2 without TLS, so it has to be asked -// explicitly. +// Both branches speak HTTP/2, which connect's bidirectional streams require and +// which Go will not do by default in either case: without TLS it has to be +// asked explicitly, and with TLS it has to be negotiated through ALPN. // -// mTLS is plan §5.6's target and replaces this; until then the stream must not -// cross an untrusted network. -func HTTP2Client(timeout time.Duration) *http.Client { - return &http.Client{ - Transport: &http2.Transport{ - AllowHTTP: true, - DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { - return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr) +// The plaintext branch exists for local runs against sim-engines. Anything +// using it must stay on a trusted network, because nothing there stops a node +// from claiming to be another one. +func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { + if !cfg.Enabled() { + return &http.Client{ + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { + return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr) + }, }, - }, + }, nil } + + // ServerName is left to the transport, which takes it from the address + // being dialled. Hardcoding it here would break the moment the control + // plane moved. + tc, err := cfg.ClientConfig("") + if err != nil { + return nil, err + } + return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil } diff --git a/internal/agent/mtls_test.go b/internal/agent/mtls_test.go new file mode 100644 index 0000000..c2f1b2a --- /dev/null +++ b/internal/agent/mtls_test.go @@ -0,0 +1,265 @@ +package agent + +import ( + "context" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +// pki issues a cluster's certificates into a temporary directory and returns +// the configurations the control plane and one agent would load. +func pki(t *testing.T, nodeID string) (server, client tlsconf.Config) { + t.Helper() + dir := t.TempDir() + + ca, err := tlsconf.NewCA("test CA") + if err != nil { + t.Fatal(err) + } + if err := ca.Write(dir); err != nil { + t.Fatal(err) + } + srv, err := ca.IssueServer("control-plane", []string{"127.0.0.1"}) + if err != nil { + t.Fatal(err) + } + if err := srv.Write(dir, "server"); err != nil { + t.Fatal(err) + } + cli, err := ca.IssueClient(nodeID) + if err != nil { + t.Fatal(err) + } + if err := cli.Write(dir, "agent"); err != nil { + t.Fatal(err) + } + + caPath := filepath.Join(dir, "ca.pem") + return tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")}, + tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "agent.pem"), Key: filepath.Join(dir, "agent-key.pem")} +} + +// startTLS runs a real service behind a real TLS listener, with identity +// required — the deployed configuration rather than an approximation of it. +func startTLS(t *testing.T, serverTLS tlsconf.Config) (*agentsvc.Service, *inventory.Store, string) { + t.Helper() + + inv := inventory.NewStore() + svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler)) + path, handler := svc.Handler() + mux := http.NewServeMux() + mux.Handle(path, handler) + + tc, err := serverTLS.ServerConfig() + if err != nil { + t.Fatal(err) + } + raw, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(mux) + srv.Listener = raw + srv.TLS = tc + srv.StartTLS() + t.Cleanup(srv.Close) + + return svc, inv, "https://" + raw.Addr().String() +} + +func runAgent(t *testing.T, cfg Config) *Agent { + t.Helper() + + cfg.MgmtAddr = "127.0.0.1" + cfg.HeartbeatInterval = 50 * time.Millisecond + cfg.ReconnectBase = 20 * time.Millisecond + cfg.ReconnectMax = 100 * time.Millisecond + cfg.Discovery.SkipGPUs = true + cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") + cfg.Superv = supervisor.Config{ + WorkDir: t.TempDir(), LogLines: 64, + StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond, + } + + client, err := Client(cfg.TLS, 5*time.Second) + if err != nil { + t.Fatal(err) + } + ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { _ = ag.Run(ctx) }() + t.Cleanup(func() { + cancel() + ag.Supervisor().StopAll(context.Background()) + }) + return ag +} + +func registered(svc *agentsvc.Service, inv *inventory.Store, nodeID string) func() bool { + return func() bool { + _, ok := inv.Get(nodeID) + return ok && len(svc.ConnectedNodes()) == 1 + } +} + +func eventually(t *testing.T, limit time.Duration, pred func() bool) bool { + t.Helper() + deadline := time.Now().Add(limit) + for time.Now().Before(deadline) { + if pred() { + return true + } + time.Sleep(10 * time.Millisecond) + } + return false +} + +func TestAgentRegistersOverMutualTLS(t *testing.T) { + serverTLS, clientTLS := pki(t, "llmgw-node01") + svc, inv, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = url + cfg.TLS = clientTLS + runAgent(t, cfg) + + if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) { + t.Fatal("the agent never registered over mutual TLS") + } +} + +func TestNodeCannotClaimAnotherNodesIdentity(t *testing.T) { + // The point of the whole exercise. node01 holds a valid certificate — it is + // a legitimate member of the cluster — and still cannot register as node02, + // because the id it claims is checked against the one it can prove. + serverTLS, clientTLS := pki(t, "llmgw-node01") + svc, inv, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node02" + cfg.Server = url + cfg.TLS = clientTLS + runAgent(t, cfg) + + // The agent reconnects on failure, so this has to stay false throughout + // rather than merely start false. + if eventually(t, 2*time.Second, func() bool { + _, ok := inv.Get("llmgw-node02") + return ok || len(svc.ConnectedNodes()) > 0 + }) { + t.Fatal("an agent registered under a node id its certificate does not authorize") + } +} + +func TestUnauthenticatedAgentIsRefused(t *testing.T) { + // The defensive half of the check: identity is required, but the connection + // carried none. In the shipped wiring TLS and this flag are tied together, + // so reaching here means someone untied them — and the answer must still be + // refusal rather than a silently trusted node id. + inv := inventory.NewStore() + svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler)) + path, handler := svc.Handler() + mux := http.NewServeMux() + mux.Handle(path, handler) + + srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{})) + t.Cleanup(srv.Close) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = srv.URL + runAgent(t, cfg) + + if eventually(t, 2*time.Second, registered(svc, inv, "llmgw-node01")) { + t.Fatal("an agent with no proven identity reached the control plane") + } +} + +func TestPlaintextAgentIsRejectedWhenTLSIsConfigured(t *testing.T) { + // A misconfiguration worth catching at startup rather than at runtime: an + // agent holding certificates but dialling http:// would connect in the + // clear and never learn that it had. + _, clientTLS := pki(t, "llmgw-node01") + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = "http://127.0.0.1:9443" + cfg.TLS = clientTLS + + if _, err := New(cfg, nil, slog.New(slog.DiscardHandler)); err == nil { + t.Fatal("an agent with certificates was allowed to dial a plaintext address") + } +} + +func TestBareAddressGainsTheRightScheme(t *testing.T) { + _, clientTLS := pki(t, "llmgw-node01") + + secure := Config{NodeID: "n", Server: "10.0.0.140:9443", TLS: clientTLS} + if err := secure.normalize(); err != nil { + t.Fatal(err) + } + if secure.Server != "https://10.0.0.140:9443" { + t.Errorf("a configured agent dials %q", secure.Server) + } + + plain := Config{NodeID: "n", Server: "10.0.0.140:9443"} + if err := plain.normalize(); err != nil { + t.Fatal(err) + } + if plain.Server != "http://10.0.0.140:9443" { + t.Errorf("an unconfigured agent dials %q", plain.Server) + } +} + +func TestRefusedAgentLearnsWhy(t *testing.T) { + // An agent that cannot register must be able to say why. Reporting a + // refusal as a clean disconnect leaves an operator with a silent reconnect + // loop and nothing to act on — and resets the backoff, so the loop is a + // fast one. + serverTLS, clientTLS := pki(t, "llmgw-node01") + _, _, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node02" + cfg.Server = url + cfg.TLS = clientTLS + cfg.Discovery.SkipGPUs = true + cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") + cfg.Superv = supervisor.Config{WorkDir: t.TempDir(), LogLines: 8} + + client, err := Client(cfg.TLS, 5*time.Second) + if err != nil { + t.Fatal(err) + } + ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + + err = ag.session(context.Background()) + if err == nil { + t.Fatal("a refused session reported success") + } + if !strings.Contains(err.Error(), "llmgw-node01") || !strings.Contains(err.Error(), "llmgw-node02") { + t.Errorf("the error names neither certificate nor claim: %v", err) + } +} diff --git a/internal/agent/shutdown_test.go b/internal/agent/shutdown_test.go new file mode 100644 index 0000000..1cd5017 --- /dev/null +++ b/internal/agent/shutdown_test.go @@ -0,0 +1,59 @@ +package agent + +import ( + "context" + "log/slog" + "path/filepath" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" +) + +// TestAgentStopsWhenContextEnds pins down something cancellation does not do by +// itself. A bidirectional stream spends its life blocked reading, and HTTP/2 +// stops watching the context once the request headers are away, so an agent +// that only cancels keeps reading until something kills it. Under systemd that +// is the full stop timeout on every restart, followed by SIGKILL. +func TestAgentStopsWhenContextEnds(t *testing.T) { + serverTLS, clientTLS := pki(t, "llmgw-node01") + svc, inv, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = url + cfg.TLS = clientTLS + cfg.HeartbeatInterval = 50 * time.Millisecond + cfg.Discovery.SkipGPUs = true + cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") + cfg.Superv = supervisor.Config{ + WorkDir: t.TempDir(), LogLines: 8, + StopTimeout: time.Second, HealthInterval: 20 * time.Millisecond, + } + + client, err := Client(cfg.TLS, 5*time.Second) + if err != nil { + t.Fatal(err) + } + ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { _ = ag.Run(ctx); close(done) }() + + // Cancelling before the stream is established would prove nothing: the + // blocked read is the case that matters. + if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) { + t.Fatal("the agent never registered") + } + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the agent did not stop within 5s of being told to") + } +} diff --git a/internal/cli/agent.go b/internal/cli/agent.go index 686c900..5d34474 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -64,7 +64,12 @@ func newAgentCmd() *cobra.Command { } cfg.Version = version.Version - a, err := agent.New(cfg, agent.HTTP2Client(30*time.Second), log) + httpClient, err := agent.Client(cfg.TLS, 30*time.Second) + if err != nil { + return err + } + + a, err := agent.New(cfg, httpClient, log) if err != nil { return err } diff --git a/internal/cli/server.go b/internal/cli/server.go index 322ccd0..1764159 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -2,6 +2,7 @@ package cli import ( "context" + "crypto/tls" "errors" "fmt" "log/slog" @@ -93,7 +94,7 @@ func newServerCmd() *cobra.Command { return err } - ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, false, log) + ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, cfg.AgentTLS.Enabled(), log) if err != nil { return err } @@ -102,7 +103,8 @@ func newServerCmd() *cobra.Command { agentPath, agentHandler = svc.Handler() log.Info("control plane running", - "models", len(cfg.Models), "agent_listen", cfg.AgentListen) + "models", len(cfg.Models), "agent_listen", cfg.AgentListen, + "agent_mtls", cfg.AgentTLS.Enabled()) } adminMux, err := rt.AdminHandler() @@ -147,17 +149,34 @@ func newServerCmd() *cobra.Command { mux := http.NewServeMux() mux.Handle(agentPath, agentHandler) agentSrv = &http.Server{ - // Agents speak plaintext HTTP/2, which Go will not do - // without being asked. mTLS replaces this (plan §5.6); - // until then the agent port must stay on a trusted network. - Handler: h2c.NewHandler(mux, &http2.Server{}), + Handler: mux, ReadHeaderTimeout: 10 * time.Second, } + if !cfg.AgentTLS.Enabled() { + // Without TLS, agents speak plaintext HTTP/2, which Go will + // not do unless asked. Nothing then stops a node from + // claiming to be another one, so this belongs on a trusted + // network only. + agentSrv.Handler = h2c.NewHandler(mux, &http2.Server{}) + } if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil { _ = apiLn.Close() _ = adminLn.Close() return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err) } + if cfg.AgentTLS.Enabled() { + tc, terr := cfg.AgentTLS.ServerConfig() + if terr != nil { + _ = apiLn.Close() + _ = adminLn.Close() + _ = agentLn.Close() + return terr + } + // Serve on an already-wrapped listener rather than + // ServeTLS: agentSrv.TLSConfig stays nil, which is exactly + // the case where net/http sets up HTTP/2 for ALPN itself. + agentLn = tls.NewListener(agentLn, tc) + } } g, gctx := errgroup.WithContext(ctx) diff --git a/internal/config/config.go b/internal/config/config.go index c6c487c..f05a83e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/knadh/koanf/v2" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Server is the whole server configuration. @@ -31,6 +32,11 @@ type Server struct { // so the fleet-facing surface can be firewalled independently. AgentListen string `koanf:"agent_listen"` + // AgentTLS authenticates the fleet-facing port. It deliberately does not + // cover the client-facing one: that surface is fronted by a publicly + // trusted certificate, while this one is mutual and private. + AgentTLS tlsconf.Config `koanf:"agent_tls"` + Router Router `koanf:"router"` Control Control `koanf:"control"` Models []catalog.Model `koanf:"models"` @@ -244,6 +250,9 @@ func (s Server) Validate() error { if s.Listen == "" { return fmt.Errorf("listen is empty") } + if err := s.AgentTLS.Validate(); err != nil { + return err + } seen := make(map[string]struct{}, len(s.Endpoints)) for i, e := range s.Endpoints { diff --git a/llmgw-plan.md b/llmgw-plan.md index 22b9669..af0bfa6 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -325,7 +325,21 @@ message ControlMessage { ``` - Commands are idempotent: a repeated `StartShard` with the same `shard_id` is a no-op. -- mTLS, with node identity taken from the certificate CN. A simple internal CA in v1; `llmgw ctl cert issue`. +- mTLS, with node identity taken from the certificate CN. The control plane + compares the CN against the node id the agent claims and refuses the stream + when they disagree, which is what makes a node's identity something it proves + rather than something it picks. +- The CA is internal, and has to be: public authorities stopped issuing the + `clientAuth` usage, so no Let's Encrypt certificate can authenticate an agent + however convenient it would be. `llmgw ctl cert init-ca | issue-server | + issue-client` runs a small authority whose key lives on one machine and whose + root is trusted everywhere. +- The server certificate must carry every address agents dial as a subject + alternative name, bare IP addresses included. Hostname verification ignores + the common name entirely, so a certificate that names the host only in its + subject is rejected by every client. +- This covers the fleet-facing port only. The client-facing API is a public + surface and is fronted by a publicly trusted certificate instead. - Three missed heartbeats mark the node `Unreachable`; after 60s it becomes `Lost` and its replicas are rescheduled. --- @@ -1250,7 +1264,10 @@ work_dir: /var/lib/llmgw-agent in CI but has not run against a driver: **verify on a GPU node before relying on the NVLink domains it reports.** - [x] connect-go stream, register/heartbeat, shard status -- [ ] mTLS — the stream is plaintext HTTP/2 today and must stay on a trusted network +- [x] mTLS: a private CA (`llmgw ctl cert`), certificates on both ends, and the + control plane refusing any agent whose claimed node id does not match its + certificate. Plaintext HTTP/2 remains the default and still must stay on a + trusted network; `agent_tls`/`tls` in the configuration turns it on. - **Exit criterion (met, via configuration rather than `ctl`):** a model starts and stops on a node. Demonstrated with real binaries: `llmgw server` and `llmgw agent` as separate processes, the agent registering, a replica being