diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 86bf11d9..efb33a06 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -598,10 +598,16 @@ type RunningShard struct { Profile string `protobuf:"bytes,10,opt,name=profile,proto3" json:"profile,omitempty"` // pool and fabric are echoed back from the StartShard that created it; the // agent has no way to derive them. - Pool string `protobuf:"bytes,11,opt,name=pool,proto3" json:"pool,omitempty"` - Fabric string `protobuf:"bytes,12,opt,name=fabric,proto3" json:"fabric,omitempty"` - LeaderAddr string `protobuf:"bytes,13,opt,name=leader_addr,json=leaderAddr,proto3" json:"leader_addr,omitempty"` - LeaderPort int32 `protobuf:"varint,14,opt,name=leader_port,json=leaderPort,proto3" json:"leader_port,omitempty"` + Pool string `protobuf:"bytes,11,opt,name=pool,proto3" json:"pool,omitempty"` + Fabric string `protobuf:"bytes,12,opt,name=fabric,proto3" json:"fabric,omitempty"` + LeaderAddr string `protobuf:"bytes,13,opt,name=leader_addr,json=leaderAddr,proto3" json:"leader_addr,omitempty"` + LeaderPort int32 `protobuf:"varint,14,opt,name=leader_port,json=leaderPort,proto3" json:"leader_port,omitempty"` + // profile_fp is the fingerprint of the profile this shard was started from, + // echoed back from StartShard. The control plane keeps no record of a running + // replica across its own restart, so without this a shard adopted after a + // restart could not be compared with the catalog — which is exactly when the + // catalog is most likely to have been edited. + ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -734,6 +740,13 @@ func (x *RunningShard) GetLeaderPort() int32 { return 0 } +func (x *RunningShard) GetProfileFp() string { + if x != nil { + return x.ProfileFp + } + return "" +} + type GPU struct { state protoimpl.MessageState `protogen:"open.v1"` Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` @@ -1318,8 +1331,11 @@ type StartShard struct { StartupTimeoutMs int64 `protobuf:"varint,12,opt,name=startup_timeout_ms,json=startupTimeoutMs,proto3" json:"startup_timeout_ms,omitempty"` // pool and fabric are control plane concepts the agent cannot derive. They // are sent so the agent can echo them back in RunningShard after a restart. - Pool string `protobuf:"bytes,13,opt,name=pool,proto3" json:"pool,omitempty"` - Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"` + Pool string `protobuf:"bytes,13,opt,name=pool,proto3" json:"pool,omitempty"` + Fabric string `protobuf:"bytes,14,opt,name=fabric,proto3" json:"fabric,omitempty"` + // profile_fp identifies the version of the profile, so the agent can say + // which one a running shard was started from. The agent never interprets it. + ProfileFp string `protobuf:"bytes,15,opt,name=profile_fp,json=profileFp,proto3" json:"profile_fp,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1452,6 +1468,13 @@ func (x *StartShard) GetFabric() string { return "" } +func (x *StartShard) GetProfileFp() string { + if x != nil { + return x.ProfileFp + } + return "" +} + type EngineProfile struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1821,7 +1844,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "free_bytes\x18\x02 \x01(\x03R\tfreeBytes\x12\x1f\n" + "\vtotal_bytes\x18\x03 \x01(\x03R\n" + "totalBytes\x12\x16\n" + - "\x06shared\x18\x04 \x01(\bR\x06shared\"\x97\x03\n" + + "\x06shared\x18\x04 \x01(\bR\x06shared\"\xb6\x03\n" + "\fRunningShard\x12\x1d\n" + "\n" + "replica_id\x18\x01 \x01(\tR\treplicaId\x12\x19\n" + @@ -1841,7 +1864,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vleader_addr\x18\r \x01(\tR\n" + "leaderAddr\x12\x1f\n" + "\vleader_port\x18\x0e \x01(\x05R\n" + - "leaderPort\"\xf9\x03\n" + + "leaderPort\x12\x1d\n" + + "\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xf9\x03\n" + "\x03GPU\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" + "\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" + @@ -1902,7 +1927,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + "\vbytes_total\x18\x03 \x01(\x03R\n" + - "bytesTotal\"\xba\x03\n" + + "bytesTotal\"\xd9\x03\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -1923,7 +1948,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aprofile\x18\v \x01(\v2\x17.llmgw.v1.EngineProfileR\aprofile\x12,\n" + "\x12startup_timeout_ms\x18\f \x01(\x03R\x10startupTimeoutMs\x12\x12\n" + "\x04pool\x18\r \x01(\tR\x04pool\x12\x16\n" + - "\x06fabric\x18\x0e \x01(\tR\x06fabric\"\xfd\x02\n" + + "\x06fabric\x18\x0e \x01(\tR\x06fabric\x12\x1d\n" + + "\n" + + "profile_fp\x18\x0f \x01(\tR\tprofileFp\"\xfd\x02\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 9bdedc10..78cab9e6 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -145,6 +145,13 @@ message RunningShard { string leader_addr = 13; int32 leader_port = 14; + + // profile_fp is the fingerprint of the profile this shard was started from, + // echoed back from StartShard. The control plane keeps no record of a running + // replica across its own restart, so without this a shard adopted after a + // restart could not be compared with the catalog — which is exactly when the + // catalog is most likely to have been edited. + string profile_fp = 15; } message GPU { @@ -286,6 +293,10 @@ message StartShard { // are sent so the agent can echo them back in RunningShard after a restart. string pool = 13; string fabric = 14; + + // profile_fp identifies the version of the profile, so the agent can say + // which one a running shard was started from. The agent never interprets it. + string profile_fp = 15; } message EngineProfile { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 6460d34f..32acfd62 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -515,6 +515,7 @@ func (a *Agent) runningShards() []*llmgwv1.RunningShard { Profile: spec.Profile.Name, Pool: spec.Pool, Fabric: spec.Fabric, + ProfileFp: spec.ProfileFP, LeaderAddr: a.cfg.MgmtAddr, LeaderPort: int32(spec.LeaderPort), }) diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 7e17f608..2712be80 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -72,6 +72,11 @@ type ShardSpec struct { // not used when building a command. Pool string Fabric string + // ProfileFP identifies the version of the profile this shard was started + // from. Carried for the same reason as Pool and Fabric, and never read: it + // lets the control plane tell, after its own restart, whether a shard that + // is still running matches the profile the catalog holds now. + ProfileFP string } // Leader reports whether this shard serves HTTP for the replica. diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 4c23a663..6094599b 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -136,6 +136,7 @@ func AdoptedFromRegister(nodeID string, shards []*llmgwv1.RunningShard) []reconc Profile: s.GetProfile(), Pool: s.GetPool(), Fabric: s.GetFabric(), + ProfileFP: s.GetProfileFp(), } if a.Rank == 0 && s.GetLeaderAddr() != "" { a.Endpoint = fmt.Sprintf("http://%s:%d", s.GetLeaderAddr(), s.GetHttpPort()) @@ -169,6 +170,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(), Pool: cmd.Pool, Fabric: cmd.Fabric, + ProfileFp: cmd.ProfileFP, } } @@ -188,6 +190,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration HTTPPort: int(m.GetHttpPort()), Pool: m.GetPool(), Fabric: m.GetFabric(), + ProfileFP: m.GetProfileFp(), Profile: engines.Profile{ Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(), TP: int(p.GetTp()), PP: int(p.GetPp()), diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 72b5731b..e906ae0c 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -30,6 +30,9 @@ type ShardCommand struct { // control plane restart (plan §6.5). Pool string Fabric string + // ProfileFP travels for the same reason, and answers a different question: + // whether the profile has been edited since this shard was started. + ProfileFP string } // Dispatcher sends commands to agents. @@ -884,6 +887,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac StartupTimeout: prof.StartupTimeout, Pool: pl.Pool, Fabric: pl.Fabric, + ProfileFP: prof.Fingerprint(), } if err := r.disp.StartShard(ctx, cmd); err != nil { r.markShard(id, s.ID, supervisor.Failed, err.Error()) @@ -1088,7 +1092,10 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad rep := &Replica{ ID: replicaID, Model: m.Name, Profile: prof.Name, Fabric: first.Fabric, Pool: first.Pool, - Weight: prof.Weight, CreatedAt: r.now(), + // From the agent, which was told it when the shard started. Empty + // from an agent too old to report it, which reads as unknown. + ProfileFP: first.ProfileFP, + Weight: prof.Weight, CreatedAt: r.now(), Endpoint: first.Endpoint, // From the profile, not from what was reported: the point is to // notice when what was reported is short. @@ -1235,6 +1242,7 @@ type AdoptedShard struct { Profile string Pool string Fabric string + ProfileFP string // Endpoint is the leader's base URL, set on the rank 0 shard. Endpoint string } diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index c0dc7ade..ebfbbc9a 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1665,3 +1665,59 @@ func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) { t.Error("a long-running idle replica was never offered for eviction") } } + +func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) { + // The fingerprint has to leave the control plane, or it cannot come back + // after a restart — and a restart is exactly when the configuration is + // likely to have been edited. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.Reconcile(t.Context()) + + started := h.disp.startedShards() + if len(started) == 0 { + t.Fatal("nothing was started") + } + // From the catalog rather than the literal, because the catalog fills in + // defaults and the fingerprint is of the profile as it is actually run. + m, ok := h.r.cat.Load().Resolve("llama") + if !ok { + t.Fatal("the model is not in the catalog") + } + want := m.Profiles[0].Fingerprint() + for _, cmd := range started { + if cmd.ProfileFP != want { + t.Errorf("shard %s carries %q, expected %q", cmd.ShardID, cmd.ProfileFP, want) + } + } +} + +func TestAnAdoptedReplicaKeepsTheProfileVersionItWasStartedWith(t *testing.T) { + // Adoption used to leave this empty, which made every replica read as + // unknown after a control plane restart — so an edited profile could never + // be noticed on the machines that were already running. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + + shard := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}) + shard.ProfileFP = "1122334455667788" + h.r.Adopt("1", []AdoptedShard{shard}) + + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("expected 1 adopted replica, got %d", len(reps)) + } + if reps[0].ProfileFP != "1122334455667788" { + t.Errorf("the agent's answer was dropped: %q", reps[0].ProfileFP) + } +} + +func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) + + // Empty, not filled in from the catalog. Copying the current profile here + // would make the replica claim to match whatever the catalog says next, + // which is the opposite of what the field is for. + if fp := h.r.Replicas()[0].ProfileFP; fp != "" { + t.Errorf("an unreported fingerprint should stay empty, got %q", fp) + } +}