From 03e38e7ddad81b1ed900e21002cdc211b5f29f4f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 23:22:44 +0200 Subject: [PATCH] Carry the profile fingerprint to the agent and back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stale-profile flag could not fire where it matters. The control plane keeps no record of a running replica across its own restart — it rebuilds them from what the agents report — and adoption left the fingerprint empty, so every replica read as unknown afterwards. That is precisely the moment the catalog is most likely to have been edited: change the configuration, restart the server, and nothing would ever say the fleet disagrees with it. The fingerprint now travels with StartShard, the agent keeps it beside the pool and fabric it already echoes back, and adoption takes the replica's version from the agent rather than from the catalog it is about to be compared against. An agent too old to report it sends nothing, which still reads as unknown rather than as matching. Co-Authored-By: Claude Opus 5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 47 ++++++++++++---- api/proto/llmgw/v1/agent.proto | 11 ++++ internal/agent/agent.go | 1 + internal/agent/engines/engine.go | 5 ++ internal/control/agentsvc/convert.go | 3 + internal/control/reconciler/reconciler.go | 10 +++- .../control/reconciler/reconciler_test.go | 56 +++++++++++++++++++ 7 files changed, 122 insertions(+), 11 deletions(-) diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 86bf11d..efb33a0 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -598,10 +598,16 @@ type RunningShard struct { Profile string `protobuf:"bytes,10,opt,name=profile,proto3" json:"profile,omitempty"` // pool and fabric are echoed back from the StartShard that created it; the // agent has no way to derive them. - Pool string `protobuf:"bytes,11,opt,name=pool,proto3" json:"pool,omitempty"` - Fabric string `protobuf:"bytes,12,opt,name=fabric,proto3" json:"fabric,omitempty"` - LeaderAddr string `protobuf:"bytes,13,opt,name=leader_addr,json=leaderAddr,proto3" json:"leader_addr,omitempty"` - LeaderPort int32 `protobuf:"varint,14,opt,name=leader_port,json=leaderPort,proto3" json:"leader_port,omitempty"` + Pool string `protobuf:"bytes,11,opt,name=pool,proto3" json:"pool,omitempty"` + Fabric string `protobuf:"bytes,12,opt,name=fabric,proto3" json:"fabric,omitempty"` + LeaderAddr string `protobuf:"bytes,13,opt,name=leader_addr,json=leaderAddr,proto3" json:"leader_addr,omitempty"` + LeaderPort int32 `protobuf:"varint,14,opt,name=leader_port,json=leaderPort,proto3" json:"leader_port,omitempty"` + // profile_fp is the fingerprint of the profile this shard was started from, + // echoed back from StartShard. The control plane keeps no record of a running + // replica across its own restart, so without this a shard adopted after a + // restart could not be compared with the catalog — which is exactly when the + // catalog is most likely to have been edited. + ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -734,6 +740,13 @@ func (x *RunningShard) GetLeaderPort() int32 { return 0 } +func (x *RunningShard) GetProfileFp() string { + if x != nil { + return x.ProfileFp + } + return "" +} + type GPU struct { state protoimpl.MessageState `protogen:"open.v1"` Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` @@ -1318,8 +1331,11 @@ type StartShard struct { StartupTimeoutMs int64 `protobuf:"varint,12,opt,name=startup_timeout_ms,json=startupTimeoutMs,proto3" json:"startup_timeout_ms,omitempty"` // pool and fabric are control plane concepts the agent cannot derive. They // are sent so the agent can echo them back in RunningShard after a restart. - Pool string `protobuf:"bytes,13,opt,name=pool,proto3" json:"pool,omitempty"` - Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"` + Pool string `protobuf:"bytes,13,opt,name=pool,proto3" json:"pool,omitempty"` + Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"` + // profile_fp identifies the version of the profile, so the agent can say + // which one a running shard was started from. The agent never interprets it. + ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1452,6 +1468,13 @@ func (x *StartShard) GetFabric() string { return "" } +func (x *StartShard) GetProfileFp() string { + if x != nil { + return x.ProfileFp + } + return "" +} + type EngineProfile struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1821,7 +1844,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "free_bytes\x18\x02 \x01(\x03R\tfreeBytes\x12\x1f\n" + "\vtotal_bytes\x18\x03 \x01(\x03R\n" + "totalBytes\x12\x16\n" + - "\x06shared\x18\x04 \x01(\bR\x06shared\"\x97\x03\n" + + "\x06shared\x18\x04 \x01(\bR\x06shared\"\xb6\x03\n" + "\fRunningShard\x12\x1d\n" + "\n" + "replica_id\x18\x01 \x01(\tR\treplicaId\x12\x19\n" + @@ -1841,7 +1864,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vleader_addr\x18\r \x01(\tR\n" + "leaderAddr\x12\x1f\n" + "\vleader_port\x18\x0e \x01(\x05R\n" + - "leaderPort\"\xf9\x03\n" + + "leaderPort\x12\x1d\n" + + "\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf9\x03\n" + "\x03GPU\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" + "\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" + @@ -1902,7 +1927,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + "\vbytes_total\x18\x03 \x01(\x03R\n" + - "bytesTotal\"\xba\x03\n" + + "bytesTotal\"\xd9\x03\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -1923,7 +1948,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aprofile\x18\v \x01(\v2\x17.llmgw.v1.EngineProfileR\aprofile\x12,\n" + "\x12startup_timeout_ms\x18\f \x01(\x03R\x10startupTimeoutMs\x12\x12\n" + "\x04pool\x18\r \x01(\tR\x04pool\x12\x16\n" + - "\x06fabric\x18\x0e \x01(\tR\x06fabric\"\xfd\x02\n" + + "\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" + + "\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xfd\x02\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 9bdedc1..78cab9e 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -145,6 +145,13 @@ message RunningShard { string leader_addr = 13; int32 leader_port = 14; + + // profile_fp is the fingerprint of the profile this shard was started from, + // echoed back from StartShard. The control plane keeps no record of a running + // replica across its own restart, so without this a shard adopted after a + // restart could not be compared with the catalog — which is exactly when the + // catalog is most likely to have been edited. + string profile_fp = 15; } message GPU { @@ -286,6 +293,10 @@ message StartShard { // are sent so the agent can echo them back in RunningShard after a restart. string pool = 13; string fabric = 14; + + // profile_fp identifies the version of the profile, so the agent can say + // which one a running shard was started from. The agent never interprets it. + string profile_fp = 15; } message EngineProfile { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 6460d34..32acfd6 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -515,6 +515,7 @@ func (a *Agent) runningShards() []*llmgwv1.RunningShard { Profile: spec.Profile.Name, Pool: spec.Pool, Fabric: spec.Fabric, + ProfileFp: spec.ProfileFP, LeaderAddr: a.cfg.MgmtAddr, LeaderPort: int32(spec.LeaderPort), }) diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 7e17f60..2712be8 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -72,6 +72,11 @@ type ShardSpec struct { // not used when building a command. Pool string Fabric string + // ProfileFP identifies the version of the profile this shard was started + // from. Carried for the same reason as Pool and Fabric, and never read: it + // lets the control plane tell, after its own restart, whether a shard that + // is still running matches the profile the catalog holds now. + ProfileFP string } // Leader reports whether this shard serves HTTP for the replica. diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 4c23a66..6094599 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -136,6 +136,7 @@ func AdoptedFromRegister(nodeID string, shards []*llmgwv1.RunningShard) []reconc Profile: s.GetProfile(), Pool: s.GetPool(), Fabric: s.GetFabric(), + ProfileFP: s.GetProfileFp(), } if a.Rank == 0 && s.GetLeaderAddr() != "" { a.Endpoint = fmt.Sprintf("http://%s:%d", s.GetLeaderAddr(), s.GetHttpPort()) @@ -169,6 +170,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(), Pool: cmd.Pool, Fabric: cmd.Fabric, + ProfileFp: cmd.ProfileFP, } } @@ -188,6 +190,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration HTTPPort: int(m.GetHttpPort()), Pool: m.GetPool(), Fabric: m.GetFabric(), + ProfileFP: m.GetProfileFp(), Profile: engines.Profile{ Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(), TP: int(p.GetTp()), PP: int(p.GetPp()), diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 72b5731..e906ae0 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -30,6 +30,9 @@ type ShardCommand struct { // control plane restart (plan §6.5). Pool string Fabric string + // ProfileFP travels for the same reason, and answers a different question: + // whether the profile has been edited since this shard was started. + ProfileFP string } // Dispatcher sends commands to agents. @@ -884,6 +887,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac StartupTimeout: prof.StartupTimeout, Pool: pl.Pool, Fabric: pl.Fabric, + ProfileFP: prof.Fingerprint(), } if err := r.disp.StartShard(ctx, cmd); err != nil { r.markShard(id, s.ID, supervisor.Failed, err.Error()) @@ -1088,7 +1092,10 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad rep := &Replica{ ID: replicaID, Model: m.Name, Profile: prof.Name, Fabric: first.Fabric, Pool: first.Pool, - Weight: prof.Weight, CreatedAt: r.now(), + // From the agent, which was told it when the shard started. Empty + // from an agent too old to report it, which reads as unknown. + ProfileFP: first.ProfileFP, + Weight: prof.Weight, CreatedAt: r.now(), Endpoint: first.Endpoint, // From the profile, not from what was reported: the point is to // notice when what was reported is short. @@ -1235,6 +1242,7 @@ type AdoptedShard struct { Profile string Pool string Fabric string + ProfileFP string // Endpoint is the leader's base URL, set on the rank 0 shard. Endpoint string } diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index c0dc7ad..ebfbbc9 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1665,3 +1665,59 @@ func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) { t.Error("a long-running idle replica was never offered for eviction") } } + +func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) { + // The fingerprint has to leave the control plane, or it cannot come back + // after a restart — and a restart is exactly when the configuration is + // likely to have been edited. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.Reconcile(t.Context()) + + started := h.disp.startedShards() + if len(started) == 0 { + t.Fatal("nothing was started") + } + // From the catalog rather than the literal, because the catalog fills in + // defaults and the fingerprint is of the profile as it is actually run. + m, ok := h.r.cat.Load().Resolve("llama") + if !ok { + t.Fatal("the model is not in the catalog") + } + want := m.Profiles[0].Fingerprint() + for _, cmd := range started { + if cmd.ProfileFP != want { + t.Errorf("shard %s carries %q, expected %q", cmd.ShardID, cmd.ProfileFP, want) + } + } +} + +func TestAnAdoptedReplicaKeepsTheProfileVersionItWasStartedWith(t *testing.T) { + // Adoption used to leave this empty, which made every replica read as + // unknown after a control plane restart — so an edited profile could never + // be noticed on the machines that were already running. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + + shard := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}) + shard.ProfileFP = "1122334455667788" + h.r.Adopt("1", []AdoptedShard{shard}) + + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("expected 1 adopted replica, got %d", len(reps)) + } + if reps[0].ProfileFP != "1122334455667788" { + t.Errorf("the agent's answer was dropped: %q", reps[0].ProfileFP) + } +} + +func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) + + // Empty, not filled in from the catalog. Copying the current profile here + // would make the replica claim to match whatever the catalog says next, + // which is the opposite of what the field is for. + if fp := h.r.Replicas()[0].ProfileFP; fp != "" { + t.Errorf("an unreported fingerprint should stay empty, got %q", fp) + } +}