package agent import ( "context" "log/slog" "net" "net/http" "net/http/httptest" "path/filepath" "strings" "testing" "time" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // pki issues a cluster's certificates into a temporary directory and returns // the configurations the control plane and one agent would load. func pki(t *testing.T, nodeID string) (server, client tlsconf.Config) { t.Helper() dir := t.TempDir() ca, err := tlsconf.NewCA("test CA") if err != nil { t.Fatal(err) } if err := ca.Write(dir); err != nil { t.Fatal(err) } srv, err := ca.IssueServer("control-plane", []string{"127.0.0.1"}) if err != nil { t.Fatal(err) } if err := srv.Write(dir, "server"); err != nil { t.Fatal(err) } cli, err := ca.IssueClient(nodeID) if err != nil { t.Fatal(err) } if err := cli.Write(dir, "agent"); err != nil { t.Fatal(err) } caPath := filepath.Join(dir, "ca.pem") return tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")}, tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "agent.pem"), Key: filepath.Join(dir, "agent-key.pem")} } // startTLS runs a real service behind a real TLS listener, with identity // required — the deployed configuration rather than an approximation of it. func startTLS(t *testing.T, serverTLS tlsconf.Config) (*agentsvc.Service, *inventory.Store, string) { t.Helper() inv := inventory.NewStore() svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler)) path, handler := svc.Handler() mux := http.NewServeMux() mux.Handle(path, handler) tc, err := serverTLS.ServerConfig() if err != nil { t.Fatal(err) } raw, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } srv := httptest.NewUnstartedServer(mux) srv.Listener = raw srv.TLS = tc srv.StartTLS() t.Cleanup(srv.Close) return svc, inv, "https://" + raw.Addr().String() } func runAgent(t *testing.T, cfg Config) *Agent { t.Helper() cfg.MgmtAddr = "127.0.0.1" cfg.HeartbeatInterval = 50 * time.Millisecond cfg.ReconnectBase = 20 * time.Millisecond cfg.ReconnectMax = 100 * time.Millisecond cfg.Discovery.SkipGPUs = true cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") cfg.Superv = supervisor.Config{ WorkDir: t.TempDir(), LogLines: 64, StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond, } client, err := Client(cfg.TLS, 5*time.Second) if err != nil { t.Fatal(err) } ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) go func() { _ = ag.Run(ctx) }() t.Cleanup(func() { cancel() ag.Supervisor().StopAll(context.Background()) }) return ag } func registered(svc *agentsvc.Service, inv *inventory.Store, nodeID string) func() bool { return func() bool { _, ok := inv.Get(nodeID) return ok && len(svc.ConnectedNodes()) == 1 } } func eventually(t *testing.T, limit time.Duration, pred func() bool) bool { t.Helper() deadline := time.Now().Add(limit) for time.Now().Before(deadline) { if pred() { return true } time.Sleep(10 * time.Millisecond) } return false } func TestAgentRegistersOverMutualTLS(t *testing.T) { serverTLS, clientTLS := pki(t, "llmgw-node01") svc, inv, url := startTLS(t, serverTLS) cfg := DefaultConfig() cfg.NodeID = "llmgw-node01" cfg.Server = url cfg.TLS = clientTLS runAgent(t, cfg) if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) { t.Fatal("the agent never registered over mutual TLS") } } func TestNodeCannotClaimAnotherNodesIdentity(t *testing.T) { // The point of the whole exercise. node01 holds a valid certificate — it is // a legitimate member of the cluster — and still cannot register as node02, // because the id it claims is checked against the one it can prove. serverTLS, clientTLS := pki(t, "llmgw-node01") svc, inv, url := startTLS(t, serverTLS) cfg := DefaultConfig() cfg.NodeID = "llmgw-node02" cfg.Server = url cfg.TLS = clientTLS runAgent(t, cfg) // The agent reconnects on failure, so this has to stay false throughout // rather than merely start false. if eventually(t, 2*time.Second, func() bool { _, ok := inv.Get("llmgw-node02") return ok || len(svc.ConnectedNodes()) > 0 }) { t.Fatal("an agent registered under a node id its certificate does not authorize") } } func TestUnauthenticatedAgentIsRefused(t *testing.T) { // The defensive half of the check: identity is required, but the connection // carried none. In the shipped wiring TLS and this flag are tied together, // so reaching here means someone untied them — and the answer must still be // refusal rather than a silently trusted node id. inv := inventory.NewStore() svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler)) path, handler := svc.Handler() mux := http.NewServeMux() mux.Handle(path, handler) srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{})) t.Cleanup(srv.Close) cfg := DefaultConfig() cfg.NodeID = "llmgw-node01" cfg.Server = srv.URL runAgent(t, cfg) if eventually(t, 2*time.Second, registered(svc, inv, "llmgw-node01")) { t.Fatal("an agent with no proven identity reached the control plane") } } func TestPlaintextAgentIsRejectedWhenTLSIsConfigured(t *testing.T) { // A misconfiguration worth catching at startup rather than at runtime: an // agent holding certificates but dialling http:// would connect in the // clear and never learn that it had. _, clientTLS := pki(t, "llmgw-node01") cfg := DefaultConfig() cfg.NodeID = "llmgw-node01" cfg.Server = "http://127.0.0.1:9443" cfg.TLS = clientTLS if _, err := New(cfg, nil, slog.New(slog.DiscardHandler)); err == nil { t.Fatal("an agent with certificates was allowed to dial a plaintext address") } } func TestBareAddressGainsTheRightScheme(t *testing.T) { _, clientTLS := pki(t, "llmgw-node01") secure := Config{NodeID: "n", Server: "10.0.0.140:9443", TLS: clientTLS} if err := secure.normalize(); err != nil { t.Fatal(err) } if secure.Server != "https://10.0.0.140:9443" { t.Errorf("a configured agent dials %q", secure.Server) } plain := Config{NodeID: "n", Server: "10.0.0.140:9443"} if err := plain.normalize(); err != nil { t.Fatal(err) } if plain.Server != "http://10.0.0.140:9443" { t.Errorf("an unconfigured agent dials %q", plain.Server) } } func TestRefusedAgentLearnsWhy(t *testing.T) { // An agent that cannot register must be able to say why. Reporting a // refusal as a clean disconnect leaves an operator with a silent reconnect // loop and nothing to act on — and resets the backoff, so the loop is a // fast one. serverTLS, clientTLS := pki(t, "llmgw-node01") _, _, url := startTLS(t, serverTLS) cfg := DefaultConfig() cfg.NodeID = "llmgw-node02" cfg.Server = url cfg.TLS = clientTLS cfg.Discovery.SkipGPUs = true cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent") cfg.Superv = supervisor.Config{WorkDir: t.TempDir(), LogLines: 8} client, err := Client(cfg.TLS, 5*time.Second) if err != nil { t.Fatal(err) } ag, err := New(cfg, client, slog.New(slog.DiscardHandler)) if err != nil { t.Fatal(err) } err = ag.session(context.Background(), ag.cfg.Server) if err == nil { t.Fatal("a refused session reported success") } if !strings.Contains(err.Error(), "llmgw-node01") || !strings.Contains(err.Error(), "llmgw-node02") { t.Errorf("the error names neither certificate nor claim: %v", err) } } // TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that // made a reconciler fix dead code in production. // // "I am running nothing" is information: it is how the control plane learns // that an engine died while its agent survived. The service used to return // early on an empty list, so that message never reached the reconciler — while // the reconciler's own tests, which call it directly, kept passing. // // This test goes through the real stream for that reason. func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) { serverTLS, clientTLS := pki(t, "llmgw-node01") inv := inventory.NewStore() rec := newRecorder() svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler)) path, handler := svc.Handler() mux := http.NewServeMux() mux.Handle(path, handler) tc, err := serverTLS.ServerConfig() if err != nil { t.Fatal(err) } raw, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } srv := httptest.NewUnstartedServer(mux) srv.Listener = raw srv.TLS = tc srv.StartTLS() t.Cleanup(srv.Close) cfg := DefaultConfig() cfg.NodeID = "llmgw-node01" cfg.Server = "https://" + raw.Addr().String() cfg.TLS = clientTLS runAgent(t, cfg) // The agent has no shards at all, which is the whole point. if !eventually(t, 10*time.Second, func() bool { return len(rec.adoptCalls()) > 0 }) { t.Fatal("an agent running nothing never reached the reconciler") } for _, call := range rec.adoptCalls() { if call.nodeID != "llmgw-node01" { t.Errorf("adopt was told node %q", call.nodeID) } } }