diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index e8d85d1..9a50a3a 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -34,6 +34,7 @@ type statusRecorder struct { ch chan status mu sync.Mutex + calls []adoptCall adopted []reconciler.AdoptedShard unclaim []string // shard ids to report back as unclaimed } @@ -53,9 +54,10 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor. } } -func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { +func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() + r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards}) r.adopted = append(r.adopted, shards...) var unclaimed []reconciler.AdoptedShard @@ -67,6 +69,19 @@ func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []rec return unclaimed } +// adoptCall records that the service reached the reconciler at all, which an +// empty shard list used to prevent. +type adoptCall struct { + nodeID string + shards []reconciler.AdoptedShard +} + +func (r *statusRecorder) adoptCalls() []adoptCall { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.calls) +} + func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/agent/mtls_test.go b/internal/agent/mtls_test.go index c2f1b2a..5b17320 100644 --- a/internal/agent/mtls_test.go +++ b/internal/agent/mtls_test.go @@ -263,3 +263,55 @@ func TestRefusedAgentLearnsWhy(t *testing.T) { t.Errorf("the error names neither certificate nor claim: %v", err) } } + +// TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that +// made a reconciler fix dead code in production. +// +// "I am running nothing" is information: it is how the control plane learns +// that an engine died while its agent survived. The service used to return +// early on an empty list, so that message never reached the reconciler — while +// the reconciler's own tests, which call it directly, kept passing. +// +// This test goes through the real stream for that reason. +func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) { + serverTLS, clientTLS := pki(t, "llmgw-node01") + + inv := inventory.NewStore() + rec := newRecorder() + svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler)) + path, handler := svc.Handler() + mux := http.NewServeMux() + mux.Handle(path, handler) + + tc, err := serverTLS.ServerConfig() + if err != nil { + t.Fatal(err) + } + raw, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(mux) + srv.Listener = raw + srv.TLS = tc + srv.StartTLS() + t.Cleanup(srv.Close) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = "https://" + raw.Addr().String() + cfg.TLS = clientTLS + runAgent(t, cfg) + + // The agent has no shards at all, which is the whole point. + if !eventually(t, 10*time.Second, func() bool { + return len(rec.adoptCalls()) > 0 + }) { + t.Fatal("an agent running nothing never reached the reconciler") + } + for _, call := range rec.adoptCalls() { + if call.nodeID != "llmgw-node01" { + t.Errorf("adopt was told node %q", call.nodeID) + } + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 25bb703..de2b7c5 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -167,10 +167,13 @@ func (s *Service) Connect( // adopt hands the agent's running shards to the reconciler and stops whatever // nothing in the catalog claims. +// +// An empty list is not skipped. "I am running nothing" is information — it is +// how the control plane learns that an engine died while its agent survived — +// and returning early on it made the reconciler's half of that exchange +// unreachable in production while its unit tests, which call the reconciler +// directly, kept passing. func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) { - if len(shards) == 0 { - return - } unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards)) for _, u := range unclaimed { // The model or profile left the catalog while this was running. Nobody