From 058ee382faa320d8821142242e74c3bba61e5266 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 10:36:35 +0200 Subject: [PATCH] An agent reporting nothing is reporting something MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit taught the reconciler to let go of shards an agent says it is not running. It made no difference on the cluster, because the service returned early when the agent's list was empty — which is precisely the case that carries the news. So the fix was dead code in production while its unit tests passed, and they passed because they called the reconciler directly. The test added here goes through the real stream instead, and asserts only that an agent running nothing reaches the reconciler at all. That is the lesson worth keeping: a test that calls the function under test directly cannot tell you the caller never calls it. Co-Authored-By: Claude Opus 5 (1M context) --- internal/agent/agent_test.go | 17 ++++++++- internal/agent/mtls_test.go | 52 ++++++++++++++++++++++++++++ internal/control/agentsvc/service.go | 9 +++-- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index e8d85d1..9a50a3a 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -34,6 +34,7 @@ type statusRecorder struct { ch chan status mu sync.Mutex + calls []adoptCall adopted []reconciler.AdoptedShard unclaim []string // shard ids to report back as unclaimed } @@ -53,9 +54,10 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor. } } -func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { +func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() + r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards}) r.adopted = append(r.adopted, shards...) var unclaimed []reconciler.AdoptedShard @@ -67,6 +69,19 @@ func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []rec return unclaimed } +// adoptCall records that the service reached the reconciler at all, which an +// empty shard list used to prevent. +type adoptCall struct { + nodeID string + shards []reconciler.AdoptedShard +} + +func (r *statusRecorder) adoptCalls() []adoptCall { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.calls) +} + func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/agent/mtls_test.go b/internal/agent/mtls_test.go index c2f1b2a..5b17320 100644 --- a/internal/agent/mtls_test.go +++ b/internal/agent/mtls_test.go @@ -263,3 +263,55 @@ func TestRefusedAgentLearnsWhy(t *testing.T) { t.Errorf("the error names neither certificate nor claim: %v", err) } } + +// TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that +// made a reconciler fix dead code in production. +// +// "I am running nothing" is information: it is how the control plane learns +// that an engine died while its agent survived. The service used to return +// early on an empty list, so that message never reached the reconciler — while +// the reconciler's own tests, which call it directly, kept passing. +// +// This test goes through the real stream for that reason. +func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) { + serverTLS, clientTLS := pki(t, "llmgw-node01") + + inv := inventory.NewStore() + rec := newRecorder() + svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler)) + path, handler := svc.Handler() + mux := http.NewServeMux() + mux.Handle(path, handler) + + tc, err := serverTLS.ServerConfig() + if err != nil { + t.Fatal(err) + } + raw, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(mux) + srv.Listener = raw + srv.TLS = tc + srv.StartTLS() + t.Cleanup(srv.Close) + + cfg := DefaultConfig() + cfg.NodeID = "llmgw-node01" + cfg.Server = "https://" + raw.Addr().String() + cfg.TLS = clientTLS + runAgent(t, cfg) + + // The agent has no shards at all, which is the whole point. + if !eventually(t, 10*time.Second, func() bool { + return len(rec.adoptCalls()) > 0 + }) { + t.Fatal("an agent running nothing never reached the reconciler") + } + for _, call := range rec.adoptCalls() { + if call.nodeID != "llmgw-node01" { + t.Errorf("adopt was told node %q", call.nodeID) + } + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 25bb703..de2b7c5 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -167,10 +167,13 @@ func (s *Service) Connect( // adopt hands the agent's running shards to the reconciler and stops whatever // nothing in the catalog claims. +// +// An empty list is not skipped. "I am running nothing" is information — it is +// how the control plane learns that an engine died while its agent survived — +// and returning early on it made the reconciler's half of that exchange +// unreachable in production while its unit tests, which call the reconciler +// directly, kept passing. func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) { - if len(shards) == 0 { - return - } unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards)) for _, u := range unclaimed { // The model or profile left the catalog while this was running. Nobody