The previous commit taught the reconciler to let go of shards an agent says it is not running. It made no difference on the cluster, because the service returned early when the agent's list was empty — which is precisely the case that carries the news. So the fix was dead code in production while its unit tests passed, and they passed because they called the reconciler directly. The test added here goes through the real stream instead, and asserts only that an agent running nothing reaches the reconciler at all. That is the lesson worth keeping: a test that calls the function under test directly cannot tell you the caller never calls it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
aef1c143c5
commit
058ee382fa
3 files changed
+74
-4
No files matched your search
@@ -34,6 +34,7 @@ type statusRecorder struct {
|
||||
ch chan status
|
||||
|
||||
mu sync.Mutex
|
||||
calls []adoptCall
|
||||
adopted []reconciler.AdoptedShard
|
||||
unclaim []string // shard ids to report back as unclaimed
|
||||
}
|
||||
@@ -53,9 +54,10 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
|
||||
}
|
||||
}
|
||||
|
||||
func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
||||
func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards})
|
||||
r.adopted = append(r.adopted, shards...)
|
||||
|
||||
var unclaimed []reconciler.AdoptedShard
|
||||
@@ -67,6 +69,19 @@ func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []rec
|
||||
return unclaimed
|
||||
}
|
||||
|
||||
// adoptCall records that the service reached the reconciler at all, which an
|
||||
// empty shard list used to prevent.
|
||||
type adoptCall struct {
|
||||
nodeID string
|
||||
shards []reconciler.AdoptedShard
|
||||
}
|
||||
|
||||
func (r *statusRecorder) adoptCalls() []adoptCall {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return slices.Clone(r.calls)
|
||||
}
|
||||
|
||||
func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -263,3 +263,55 @@ func TestRefusedAgentLearnsWhy(t *testing.T) {
|
||||
t.Errorf("the error names neither certificate nor claim: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that
|
||||
// made a reconciler fix dead code in production.
|
||||
//
|
||||
// "I am running nothing" is information: it is how the control plane learns
|
||||
// that an engine died while its agent survived. The service used to return
|
||||
// early on an empty list, so that message never reached the reconciler — while
|
||||
// the reconciler's own tests, which call it directly, kept passing.
|
||||
//
|
||||
// This test goes through the real stream for that reason.
|
||||
func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) {
|
||||
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||
|
||||
inv := inventory.NewStore()
|
||||
rec := newRecorder()
|
||||
svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler))
|
||||
path, handler := svc.Handler()
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(path, handler)
|
||||
|
||||
tc, err := serverTLS.ServerConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewUnstartedServer(mux)
|
||||
srv.Listener = raw
|
||||
srv.TLS = tc
|
||||
srv.StartTLS()
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "llmgw-node01"
|
||||
cfg.Server = "https://" + raw.Addr().String()
|
||||
cfg.TLS = clientTLS
|
||||
runAgent(t, cfg)
|
||||
|
||||
// The agent has no shards at all, which is the whole point.
|
||||
if !eventually(t, 10*time.Second, func() bool {
|
||||
return len(rec.adoptCalls()) > 0
|
||||
}) {
|
||||
t.Fatal("an agent running nothing never reached the reconciler")
|
||||
}
|
||||
for _, call := range rec.adoptCalls() {
|
||||
if call.nodeID != "llmgw-node01" {
|
||||
t.Errorf("adopt was told node %q", call.nodeID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -167,10 +167,13 @@ func (s *Service) Connect(
|
||||
|
||||
// adopt hands the agent's running shards to the reconciler and stops whatever
|
||||
// nothing in the catalog claims.
|
||||
//
|
||||
// An empty list is not skipped. "I am running nothing" is information — it is
|
||||
// how the control plane learns that an engine died while its agent survived —
|
||||
// and returning early on it made the reconciler's half of that exchange
|
||||
// unreachable in production while its unit tests, which call the reconciler
|
||||
// directly, kept passing.
|
||||
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
|
||||
if len(shards) == 0 {
|
||||
return
|
||||
}
|
||||
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
|
||||
for _, u := range unclaimed {
|
||||
// The model or profile left the catalog while this was running. Nobody
|
||||
|
||||
Reference in new issue
Block a user