diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml index 50ed81e..a78f521 100644 --- a/deploy/examples/agent.yaml +++ b/deploy/examples/agent.yaml @@ -7,6 +7,21 @@ node_id: gpu-a-01 # defaults to the hostname server: "http://llmgw-server.internal:7443" +# The agent's half of mutual TLS. The certificate's common name must be this +# node's id, because the control plane compares the two and refuses the stream +# when they disagree: +# +# llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki +# +# With this set, `server` above must be an https:// address — an agent holding +# certificates and dialling http:// is refused at startup rather than connecting +# in the clear without noticing. +# +# tls: +# ca: /etc/llmgw/ca.pem +# cert: /etc/llmgw/agent.pem +# key: /etc/llmgw/agent-key.pem + # The address engines are reachable on, and the address NCCL bootstraps over. # Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA. mgmt_addr: "10.0.0.11" diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 439587e..e2668db 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -9,10 +9,33 @@ listen: ":8080" # client-facing OpenAI API admin_listen: ":9090" # metrics, health, pprof — do not expose this publicly # Where node agents connect. Separate from the client port so the fleet-facing -# surface can be firewalled independently. Agents speak plaintext HTTP/2 today, -# so this must stay on a trusted network until mTLS lands (plan §5.6). +# surface can be firewalled independently. agent_listen: ":7443" +# Mutual TLS on the agent port (plan §5.6). With it, a node's identity is taken +# from its certificate's common name and an agent claiming a different node id +# is refused. Without it the port is plaintext HTTP/2 and anyone who can reach +# it can claim to be any node, so it must stay on a trusted network. +# +# These certificates come from llmgw's own authority, not a public one: public +# authorities stopped issuing the clientAuth usage, so no Let's Encrypt +# certificate can authenticate an agent. +# +# llmgw ctl cert init-ca --dir /etc/llmgw/pki +# llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.140 --dir /etc/llmgw/pki +# +# Every address an agent might dial has to appear in --host, bare IP addresses +# included: hostname verification reads subject alternative names and ignores +# the common name entirely. +# +# This covers the fleet-facing port only. The client-facing API above is public +# and belongs behind a publicly trusted certificate. +# +# agent_tls: +# ca: /etc/llmgw/ca.pem +# cert: /etc/llmgw/server.pem +# key: /etc/llmgw/server-key.pem + router: scrape_interval: 2s scrape_timeout: 1s diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 2b671bd..d21e0f7 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -11,6 +11,7 @@ import ( "math/rand/v2" "net" "net/http" + "strings" "sync" "time" @@ -23,6 +24,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Config configures the agent. @@ -45,6 +47,10 @@ type Config struct { ReconnectBase time.Duration `koanf:"reconnect_base"` ReconnectMax time.Duration `koanf:"reconnect_max"` + // TLS authenticates both ends of the control-plane stream. Leaving it unset + // keeps the plaintext behaviour, which only belongs on a trusted network. + TLS tlsconf.Config `koanf:"tls"` + Discovery discovery.Options `koanf:"discovery"` Engines engines.Config `koanf:"engines"` Superv supervisor.Config `koanf:"supervisor"` @@ -73,6 +79,12 @@ func (c *Config) normalize() error { if c.Server == "" { return errors.New("server is required") } + if err := c.TLS.Validate(); err != nil { + return err + } + if err := c.normalizeServerURL(); err != nil { + return err + } if c.MgmtInterface == "" { c.MgmtInterface = d.MgmtInterface } @@ -88,6 +100,31 @@ func (c *Config) normalize() error { return nil } +// normalizeServerURL makes the configured address a URL connect can use, and +// refuses the combination that would quietly undo the certificates: a scheme +// that says plaintext while TLS is configured. Silently connecting anyway would +// mean a cluster believing it has mutual authentication when it has none. +func (c *Config) normalizeServerURL() error { + secure := c.TLS.Enabled() + switch { + case strings.HasPrefix(c.Server, "https://"): + if !secure { + return errors.New("server is an https:// address but no tls certificates are configured") + } + case strings.HasPrefix(c.Server, "http://"): + if secure { + return errors.New("tls is configured but server is an http:// address; use https://") + } + case strings.Contains(c.Server, "://"): + return fmt.Errorf("server %q must be an http:// or https:// address", c.Server) + case secure: + c.Server = "https://" + c.Server + default: + c.Server = "http://" + c.Server + } + return nil +} + // Agent runs on a GPU node. type Agent struct { cfg Config @@ -286,6 +323,10 @@ func (a *Agent) scan(ctx context.Context) error { func (a *Agent) session(ctx context.Context) error { client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server) + // parent is kept because the derived context is cancelled from inside this + // function; asking the derived one whether we are shutting down would + // always say yes, and every stream error would read as a clean close. + parent := ctx ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -310,6 +351,22 @@ func (a *Agent) session(ctx context.Context) error { return fmt.Errorf("registering: %w", err) } + // Closing the response is what actually unblocks Receive below. Cancelling + // the context is not enough: HTTP/2 watches it while the request body is + // being written, and a bidirectional stream spends its life blocked on the + // read side instead. Without this an agent told to stop keeps reading until + // something kills it — which for systemd means the full stop timeout on + // every restart. + watchDone := make(chan struct{}) + defer close(watchDone) + go func() { + select { + case <-ctx.Done(): + _ = stream.CloseResponse() + case <-watchDone: + } + }() + // Sends are serialized through this channel: connect streams are not safe // for concurrent writes, and heartbeats race with status reports. out := make(chan *llmgwv1.AgentMessage, 32) @@ -325,9 +382,18 @@ func (a *Agent) session(ctx context.Context) error { msg, err := stream.Receive() if err != nil { cancel() - if errors.Is(err, io.EOF) || ctx.Err() != nil { + switch { + case parent.Err() != nil: + // The agent is stopping; the stream ending is the consequence. + return nil + case errors.Is(err, io.EOF): + // The control plane closed the stream, which a restart does. + // Reconnect immediately rather than backing off. return nil } + // Everything else is worth both reporting and backing off from. A + // refused certificate arrives here, and an agent that cannot say + // why it is not registering is one an operator cannot fix. return err } a.handle(ctx, msg, out) @@ -536,23 +602,33 @@ func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo { return out } -// HTTP2Client returns the client the agent should use to reach the control -// plane over plaintext HTTP/2. +// Client returns the client the agent reaches the control plane with. // -// connect's bidirectional streams require HTTP/2, and an agent talking to a -// control plane over a trusted network has no TLS to negotiate it with. Go's -// default transport will not speak HTTP/2 without TLS, so it has to be asked -// explicitly. +// Both branches speak HTTP/2, which connect's bidirectional streams require and +// which Go will not do by default in either case: without TLS it has to be +// asked explicitly, and with TLS it has to be negotiated through ALPN. // -// mTLS is plan §5.6's target and replaces this; until then the stream must not -// cross an untrusted network. -func HTTP2Client(timeout time.Duration) *http.Client { - return &http.Client{ - Transport: &http2.Transport{ - AllowHTTP: true, - DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { - return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr) +// The plaintext branch exists for local runs against sim-engines. Anything +// using it must stay on a trusted network, because nothing there stops a node +// from claiming to be another one. +func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { + if !cfg.Enabled() { + return &http.Client{ + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { + return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr) + }, }, - }, + }, nil } + + // ServerName is left to the transport, which takes it from the address + // being dialled. Hardcoding it here would break the moment the control + // plane moved. + tc, err := cfg.ClientConfig("") + if err != nil { + return nil, err + } + return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil } diff --git a/internal/agent/mtls_test.go b/internal/agent/mtls_test.go new file mode 100644 index 0000000..c2f1b2a --- /dev/null +++ b/internal/agent/mtls_test.go @@ -0,0 +1,265 @@ +package agent + +import ( + "context" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +// pki issues a cluster's certificates into a temporary directory and returns +// the configurations the control plane and one agent would load. +func pki(t *testing.T, nodeID string) (server, client tlsconf.Config) { + t.Helper() + dir := t.TempDir() + + ca, err := tlsconf.NewCA("test CA") + if err != nil { + t.Fatal(err) + } + if err := ca.Write(dir); err != nil { + t.Fatal(err) + } + srv, err := ca.IssueServer("control-plane", []string{"127.0.0.1"}) + if err != nil { + t.Fatal(err) + } + if err := srv.Write(dir, "server"); err != nil { + t.Fatal(err) + } + cli, err := ca.IssueClient(nodeID) + if err != nil { + t.Fatal(err) + } + if err := cli.Write(dir, "agent"); err != nil { + t.Fatal(err) + } + + caPath := filepath.Join(dir, "ca.pem") + return tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")}, + tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "agent.pem"), Key: filepath.Join(dir, "agent-key.pem")} +} + +// startTLS runs a real service behind a real TLS listener, with identity +// required — the deployed configuration rather than an approximation of it. +func startTLS(t *testing.T, serverTLS tlsconf.Config) (*agentsvc.Service, *inventory.Store, string) { + t.Helper() + + inv := inventory.NewStore() + svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler)) + path, handler := svc.Handler() + mux := http.NewServeMux() + mux.Handle(path, handler) + + tc, err := serverTLS.ServerConfig() + if err != nil { + t.Fatal(err) + } + raw, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(mux) + srv.Listener = raw + srv.TLS = tc + srv.StartTLS() + t.Cleanup(srv.Close) + + return svc, inv, "https://" + raw.Addr().String() +} + +func runAgent(t *testing.T, cfg Config) *Agent { + t.Helper() + + cfg.MgmtAddr = "127.0.0.1" + cfg.HeartbeatInterval = 50 * time.Millisecond + cfg.ReconnectBase = 20 * time.Millisecond + cfg.ReconnectMax = 100 * time.Millisecond + cfg.Discovery.SkipGPUs = true + cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") + cfg.Superv = supervisor.Config{ + WorkDir: t.TempDir(), LogLines: 64, + StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond, + } + + client, err := Client(cfg.TLS, 5*time.Second) + if err != nil { + t.Fatal(err) + } + ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { _ = ag.Run(ctx) }() + t.Cleanup(func() { + cancel() + ag.Supervisor().StopAll(context.Background()) + }) + return ag +} + +func registered(svc *agentsvc.Service, inv *inventory.Store, nodeID string) func() bool { + return func() bool { + _, ok := inv.Get(nodeID) + return ok && len(svc.ConnectedNodes()) == 1 + } +} + +func eventually(t *testing.T, limit time.Duration, pred func() bool) bool { + t.Helper() + deadline := time.Now().Add(limit) + for time.Now().Before(deadline) { + if pred() { + return true + } + time.Sleep(10 * time.Millisecond) + } + return false +} + +func TestAgentRegistersOverMutualTLS(t *testing.T) { + serverTLS, clientTLS := pki(t, "llmgw-node01") + svc, inv, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = url + cfg.TLS = clientTLS + runAgent(t, cfg) + + if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) { + t.Fatal("the agent never registered over mutual TLS") + } +} + +func TestNodeCannotClaimAnotherNodesIdentity(t *testing.T) { + // The point of the whole exercise. node01 holds a valid certificate — it is + // a legitimate member of the cluster — and still cannot register as node02, + // because the id it claims is checked against the one it can prove. + serverTLS, clientTLS := pki(t, "llmgw-node01") + svc, inv, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node02" + cfg.Server = url + cfg.TLS = clientTLS + runAgent(t, cfg) + + // The agent reconnects on failure, so this has to stay false throughout + // rather than merely start false. + if eventually(t, 2*time.Second, func() bool { + _, ok := inv.Get("llmgw-node02") + return ok || len(svc.ConnectedNodes()) > 0 + }) { + t.Fatal("an agent registered under a node id its certificate does not authorize") + } +} + +func TestUnauthenticatedAgentIsRefused(t *testing.T) { + // The defensive half of the check: identity is required, but the connection + // carried none. In the shipped wiring TLS and this flag are tied together, + // so reaching here means someone untied them — and the answer must still be + // refusal rather than a silently trusted node id. + inv := inventory.NewStore() + svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler)) + path, handler := svc.Handler() + mux := http.NewServeMux() + mux.Handle(path, handler) + + srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{})) + t.Cleanup(srv.Close) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = srv.URL + runAgent(t, cfg) + + if eventually(t, 2*time.Second, registered(svc, inv, "llmgw-node01")) { + t.Fatal("an agent with no proven identity reached the control plane") + } +} + +func TestPlaintextAgentIsRejectedWhenTLSIsConfigured(t *testing.T) { + // A misconfiguration worth catching at startup rather than at runtime: an + // agent holding certificates but dialling http:// would connect in the + // clear and never learn that it had. + _, clientTLS := pki(t, "llmgw-node01") + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = "http://127.0.0.1:9443" + cfg.TLS = clientTLS + + if _, err := New(cfg, nil, slog.New(slog.DiscardHandler)); err == nil { + t.Fatal("an agent with certificates was allowed to dial a plaintext address") + } +} + +func TestBareAddressGainsTheRightScheme(t *testing.T) { + _, clientTLS := pki(t, "llmgw-node01") + + secure := Config{NodeID: "n", Server: "10.0.0.140:9443", TLS: clientTLS} + if err := secure.normalize(); err != nil { + t.Fatal(err) + } + if secure.Server != "https://10.0.0.140:9443" { + t.Errorf("a configured agent dials %q", secure.Server) + } + + plain := Config{NodeID: "n", Server: "10.0.0.140:9443"} + if err := plain.normalize(); err != nil { + t.Fatal(err) + } + if plain.Server != "http://10.0.0.140:9443" { + t.Errorf("an unconfigured agent dials %q", plain.Server) + } +} + +func TestRefusedAgentLearnsWhy(t *testing.T) { + // An agent that cannot register must be able to say why. Reporting a + // refusal as a clean disconnect leaves an operator with a silent reconnect + // loop and nothing to act on — and resets the backoff, so the loop is a + // fast one. + serverTLS, clientTLS := pki(t, "llmgw-node01") + _, _, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node02" + cfg.Server = url + cfg.TLS = clientTLS + cfg.Discovery.SkipGPUs = true + cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") + cfg.Superv = supervisor.Config{WorkDir: t.TempDir(), LogLines: 8} + + client, err := Client(cfg.TLS, 5*time.Second) + if err != nil { + t.Fatal(err) + } + ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + + err = ag.session(context.Background()) + if err == nil { + t.Fatal("a refused session reported success") + } + if !strings.Contains(err.Error(), "llmgw-node01") || !strings.Contains(err.Error(), "llmgw-node02") { + t.Errorf("the error names neither certificate nor claim: %v", err) + } +} diff --git a/internal/agent/shutdown_test.go b/internal/agent/shutdown_test.go new file mode 100644 index 0000000..1cd5017 --- /dev/null +++ b/internal/agent/shutdown_test.go @@ -0,0 +1,59 @@ +package agent + +import ( + "context" + "log/slog" + "path/filepath" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" +) + +// TestAgentStopsWhenContextEnds pins down something cancellation does not do by +// itself. A bidirectional stream spends its life blocked reading, and HTTP/2 +// stops watching the context once the request headers are away, so an agent +// that only cancels keeps reading until something kills it. Under systemd that +// is the full stop timeout on every restart, followed by SIGKILL. +func TestAgentStopsWhenContextEnds(t *testing.T) { + serverTLS, clientTLS := pki(t, "llmgw-node01") + svc, inv, url := startTLS(t, serverTLS) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = url + cfg.TLS = clientTLS + cfg.HeartbeatInterval = 50 * time.Millisecond + cfg.Discovery.SkipGPUs = true + cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") + cfg.Superv = supervisor.Config{ + WorkDir: t.TempDir(), LogLines: 8, + StopTimeout: time.Second, HealthInterval: 20 * time.Millisecond, + } + + client, err := Client(cfg.TLS, 5*time.Second) + if err != nil { + t.Fatal(err) + } + ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { _ = ag.Run(ctx); close(done) }() + + // Cancelling before the stream is established would prove nothing: the + // blocked read is the case that matters. + if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) { + t.Fatal("the agent never registered") + } + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the agent did not stop within 5s of being told to") + } +} diff --git a/internal/cli/agent.go b/internal/cli/agent.go index 686c900..5d34474 100644 --- a/internal/cli/agent.go +++ b/internal/cli/agent.go @@ -64,7 +64,12 @@ func newAgentCmd() *cobra.Command { } cfg.Version = version.Version - a, err := agent.New(cfg, agent.HTTP2Client(30*time.Second), log) + httpClient, err := agent.Client(cfg.TLS, 30*time.Second) + if err != nil { + return err + } + + a, err := agent.New(cfg, httpClient, log) if err != nil { return err } diff --git a/internal/cli/server.go b/internal/cli/server.go index 322ccd0..1764159 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -2,6 +2,7 @@ package cli import ( "context" + "crypto/tls" "errors" "fmt" "log/slog" @@ -93,7 +94,7 @@ func newServerCmd() *cobra.Command { return err } - ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, false, log) + ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, cfg.AgentTLS.Enabled(), log) if err != nil { return err } @@ -102,7 +103,8 @@ func newServerCmd() *cobra.Command { agentPath, agentHandler = svc.Handler() log.Info("control plane running", - "models", len(cfg.Models), "agent_listen", cfg.AgentListen) + "models", len(cfg.Models), "agent_listen", cfg.AgentListen, + "agent_mtls", cfg.AgentTLS.Enabled()) } adminMux, err := rt.AdminHandler() @@ -147,17 +149,34 @@ func newServerCmd() *cobra.Command { mux := http.NewServeMux() mux.Handle(agentPath, agentHandler) agentSrv = &http.Server{ - // Agents speak plaintext HTTP/2, which Go will not do - // without being asked. mTLS replaces this (plan §5.6); - // until then the agent port must stay on a trusted network. - Handler: h2c.NewHandler(mux, &http2.Server{}), + Handler: mux, ReadHeaderTimeout: 10 * time.Second, } + if !cfg.AgentTLS.Enabled() { + // Without TLS, agents speak plaintext HTTP/2, which Go will + // not do unless asked. Nothing then stops a node from + // claiming to be another one, so this belongs on a trusted + // network only. + agentSrv.Handler = h2c.NewHandler(mux, &http2.Server{}) + } if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil { _ = apiLn.Close() _ = adminLn.Close() return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err) } + if cfg.AgentTLS.Enabled() { + tc, terr := cfg.AgentTLS.ServerConfig() + if terr != nil { + _ = apiLn.Close() + _ = adminLn.Close() + _ = agentLn.Close() + return terr + } + // Serve on an already-wrapped listener rather than + // ServeTLS: agentSrv.TLSConfig stays nil, which is exactly + // the case where net/http sets up HTTP/2 for ALPN itself. + agentLn = tls.NewListener(agentLn, tc) + } } g, gctx := errgroup.WithContext(ctx) diff --git a/internal/config/config.go b/internal/config/config.go index c6c487c..f05a83e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/knadh/koanf/v2" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Server is the whole server configuration. @@ -31,6 +32,11 @@ type Server struct { // so the fleet-facing surface can be firewalled independently. AgentListen string `koanf:"agent_listen"` + // AgentTLS authenticates the fleet-facing port. It deliberately does not + // cover the client-facing one: that surface is fronted by a publicly + // trusted certificate, while this one is mutual and private. + AgentTLS tlsconf.Config `koanf:"agent_tls"` + Router Router `koanf:"router"` Control Control `koanf:"control"` Models []catalog.Model `koanf:"models"` @@ -244,6 +250,9 @@ func (s Server) Validate() error { if s.Listen == "" { return fmt.Errorf("listen is empty") } + if err := s.AgentTLS.Validate(); err != nil { + return err + } seen := make(map[string]struct{}, len(s.Endpoints)) for i, e := range s.Endpoints { diff --git a/llmgw-plan.md b/llmgw-plan.md index 22b9669..af0bfa6 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -325,7 +325,21 @@ message ControlMessage { ``` - Commands are idempotent: a repeated `StartShard` with the same `shard_id` is a no-op. -- mTLS, with node identity taken from the certificate CN. A simple internal CA in v1; `llmgw ctl cert issue`. +- mTLS, with node identity taken from the certificate CN. The control plane + compares the CN against the node id the agent claims and refuses the stream + when they disagree, which is what makes a node's identity something it proves + rather than something it picks. +- The CA is internal, and has to be: public authorities stopped issuing the + `clientAuth` usage, so no Let's Encrypt certificate can authenticate an agent + however convenient it would be. `llmgw ctl cert init-ca | issue-server | + issue-client` runs a small authority whose key lives on one machine and whose + root is trusted everywhere. +- The server certificate must carry every address agents dial as a subject + alternative name, bare IP addresses included. Hostname verification ignores + the common name entirely, so a certificate that names the host only in its + subject is rejected by every client. +- This covers the fleet-facing port only. The client-facing API is a public + surface and is fronted by a publicly trusted certificate instead. - Three missed heartbeats mark the node `Unreachable`; after 60s it becomes `Lost` and its replicas are rescheduled. --- @@ -1250,7 +1264,10 @@ work_dir: /var/lib/llmgw-agent in CI but has not run against a driver: **verify on a GPU node before relying on the NVLink domains it reports.** - [x] connect-go stream, register/heartbeat, shard status -- [ ] mTLS — the stream is plaintext HTTP/2 today and must stay on a trusted network +- [x] mTLS: a private CA (`llmgw ctl cert`), certificates on both ends, and the + control plane refusing any agent whose claimed node id does not match its + certificate. Plaintext HTTP/2 remains the default and still must stay on a + trusted network; `agent_tls`/`tls` in the configuration turns it on. - **Exit criterion (met, via configuration rather than `ctl`):** a model starts and stops on a node. Demonstrated with real binaries: `llmgw server` and `llmgw agent` as separate processes, the agent registering, a replica being