Carry the profile fingerprint to the agent and back

The stale-profile flag could not fire where it matters. The control plane
keeps no record of a running replica across its own restart — it rebuilds
them from what the agents report — and adoption left the fingerprint empty,
so every replica read as unknown afterwards. That is precisely the moment
the catalog is most likely to have been edited: change the configuration,
restart the server, and nothing would ever say the fleet disagrees with it.

The fingerprint now travels with StartShard, the agent keeps it beside the
pool and fabric it already echoes back, and adoption takes the replica's
version from the agent rather than from the catalog it is about to be
compared against. An agent too old to report it sends nothing, which still
reads as unknown rather than as matching.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 23:22:44 +02:00
1 parent 40f5c930c4
commit 03e38e7dda
7 files changed
+122 -11

No files matched your search

+3
View File
@@ -136,6 +136,7 @@ func AdoptedFromRegister(nodeID string, shards []*llmgwv1.RunningShard) []reconc
Profile: s.GetProfile(),
Pool: s.GetPool(),
Fabric: s.GetFabric(),
ProfileFP: s.GetProfileFp(),
}
if a.Rank == 0 && s.GetLeaderAddr() != "" {
a.Endpoint = fmt.Sprintf("http://%s:%d", s.GetLeaderAddr(), s.GetHttpPort())
@@ -169,6 +170,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
Pool: cmd.Pool,
Fabric: cmd.Fabric,
ProfileFp: cmd.ProfileFP,
}
}
@@ -188,6 +190,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
HTTPPort: int(m.GetHttpPort()),
Pool: m.GetPool(),
Fabric: m.GetFabric(),
ProfileFP: m.GetProfileFp(),
Profile: engines.Profile{
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
TP: int(p.GetTp()), PP: int(p.GetPp()),
+9 -1
View File
@@ -30,6 +30,9 @@ type ShardCommand struct {
// control plane restart (plan §6.5).
Pool string
Fabric string
// ProfileFP travels for the same reason, and answers a different question:
// whether the profile has been edited since this shard was started.
ProfileFP string
}
// Dispatcher sends commands to agents.
@@ -884,6 +887,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
Fabric: pl.Fabric,
ProfileFP: prof.Fingerprint(),
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
@@ -1088,7 +1092,10 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
rep := &Replica{
ID: replicaID, Model: m.Name, Profile: prof.Name,
Fabric: first.Fabric, Pool: first.Pool,
Weight: prof.Weight, CreatedAt: r.now(),
// From the agent, which was told it when the shard started. Empty
// from an agent too old to report it, which reads as unknown.
ProfileFP: first.ProfileFP,
Weight: prof.Weight, CreatedAt: r.now(),
Endpoint: first.Endpoint,
// From the profile, not from what was reported: the point is to
// notice when what was reported is short.
@@ -1235,6 +1242,7 @@ type AdoptedShard struct {
Profile string
Pool string
Fabric string
ProfileFP string
// Endpoint is the leader's base URL, set on the rank 0 shard.
Endpoint string
}
@@ -1665,3 +1665,59 @@ func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
t.Error("a long-running idle replica was never offered for eviction")
}
}
func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) {
// The fingerprint has to leave the control plane, or it cannot come back
// after a restart — and a restart is exactly when the configuration is
// likely to have been edited.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
if len(started) == 0 {
t.Fatal("nothing was started")
}
// From the catalog rather than the literal, because the catalog fills in
// defaults and the fingerprint is of the profile as it is actually run.
m, ok := h.r.cat.Load().Resolve("llama")
if !ok {
t.Fatal("the model is not in the catalog")
}
want := m.Profiles[0].Fingerprint()
for _, cmd := range started {
if cmd.ProfileFP != want {
t.Errorf("shard %s carries %q, expected %q", cmd.ShardID, cmd.ProfileFP, want)
}
}
}
func TestAnAdoptedReplicaKeepsTheProfileVersionItWasStartedWith(t *testing.T) {
// Adoption used to leave this empty, which made every replica read as
// unknown after a control plane restart — so an edited profile could never
// be noticed on the machines that were already running.
h := newHarness(t, testModel(1), gpuNode("1", 8))
shard := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})
shard.ProfileFP = "1122334455667788"
h.r.Adopt("1", []AdoptedShard{shard})
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
}
if reps[0].ProfileFP != "1122334455667788" {
t.Errorf("the agent's answer was dropped: %q", reps[0].ProfileFP)
}
}
func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
// Empty, not filled in from the catalog. Copying the current profile here
// would make the replica claim to match whatever the catalog says next,
// which is the opposite of what the field is for.
if fp := h.r.Replicas()[0].ProfileFP; fp != "" {
t.Errorf("an unreported fingerprint should stay empty, got %q", fp)
}
}