An agent reporting nothing is reporting something
CI / test (push) Successful in 1m18s

The previous commit taught the reconciler to let go of shards an agent says
it is not running. It made no difference on the cluster, because the service
returned early when the agent's list was empty — which is precisely the case
that carries the news.

So the fix was dead code in production while its unit tests passed, and they
passed because they called the reconciler directly. The test added here goes
through the real stream instead, and asserts only that an agent running
nothing reaches the reconciler at all.

That is the lesson worth keeping: a test that calls the function under test
directly cannot tell you the caller never calls it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 10:36:35 +02:00
1 parent aef1c143c5
commit 058ee382fa
3 files changed
+74 -4

No files matched your search

+16 -1
View File
@@ -34,6 +34,7 @@ type statusRecorder struct {
ch chan status
mu sync.Mutex
calls []adoptCall
adopted []reconciler.AdoptedShard
unclaim []string // shard ids to report back as unclaimed
}
@@ -53,9 +54,10 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
}
}
func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
r.mu.Lock()
defer r.mu.Unlock()
r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards})
r.adopted = append(r.adopted, shards...)
var unclaimed []reconciler.AdoptedShard
@@ -67,6 +69,19 @@ func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []rec
return unclaimed
}
// adoptCall records that the service reached the reconciler at all, which an
// empty shard list used to prevent.
type adoptCall struct {
nodeID string
shards []reconciler.AdoptedShard
}
func (r *statusRecorder) adoptCalls() []adoptCall {
r.mu.Lock()
defer r.mu.Unlock()
return slices.Clone(r.calls)
}
func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard {
r.mu.Lock()
defer r.mu.Unlock()
+52
View File
@@ -263,3 +263,55 @@ func TestRefusedAgentLearnsWhy(t *testing.T) {
t.Errorf("the error names neither certificate nor claim: %v", err)
}
}
// TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that
// made a reconciler fix dead code in production.
//
// "I am running nothing" is information: it is how the control plane learns
// that an engine died while its agent survived. The service used to return
// early on an empty list, so that message never reached the reconciler — while
// the reconciler's own tests, which call it directly, kept passing.
//
// This test goes through the real stream for that reason.
func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) {
serverTLS, clientTLS := pki(t, "llmgw-node01")
inv := inventory.NewStore()
rec := newRecorder()
svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler))
path, handler := svc.Handler()
mux := http.NewServeMux()
mux.Handle(path, handler)
tc, err := serverTLS.ServerConfig()
if err != nil {
t.Fatal(err)
}
raw, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
srv := httptest.NewUnstartedServer(mux)
srv.Listener = raw
srv.TLS = tc
srv.StartTLS()
t.Cleanup(srv.Close)
cfg := DefaultConfig()
cfg.NodeID = "llmgw-node01"
cfg.Server = "https://" + raw.Addr().String()
cfg.TLS = clientTLS
runAgent(t, cfg)
// The agent has no shards at all, which is the whole point.
if !eventually(t, 10*time.Second, func() bool {
return len(rec.adoptCalls()) > 0
}) {
t.Fatal("an agent running nothing never reached the reconciler")
}
for _, call := range rec.adoptCalls() {
if call.nodeID != "llmgw-node01" {
t.Errorf("adopt was told node %q", call.nodeID)
}
}
}
+6 -3
View File
@@ -167,10 +167,13 @@ func (s *Service) Connect(
// adopt hands the agent's running shards to the reconciler and stops whatever
// nothing in the catalog claims.
//
// An empty list is not skipped. "I am running nothing" is information — it is
// how the control plane learns that an engine died while its agent survived —
// and returning early on it made the reconciler's half of that exchange
// unreachable in production while its unit tests, which call the reconciler
// directly, kept passing.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
if len(shards) == 0 {
return
}
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody