From a35e8796d22fdfcd3247cc365a3dbe3117225e66 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 07:49:08 +0200 Subject: [PATCH] Carry a download's progress to where somebody is looking at it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent has always logged a percentage as it fetches weights, with a comment saying a replica pulling a hundred gigabytes and one that is wedged look identical from the control plane. They still did: the log is on the node, and the node is not where anyone is looking when a replica has been starting for ten minutes. Progress now rides on the shard status and appears under the replica in `ctl replicas` — a line rather than a column, because most replicas are not downloading anything and a column would be empty for all of them. It is deliberately not a state transition. Treating each chunk as one would re-mark the shard as starting on every report and reset the timer that decides when a slow start has become a failed one, so an hour-long download would keep a doomed replica alive indefinitely. It arrives on its own method for that reason, and a real transition clears it — otherwise a finished download sits in the listing at ninety-nine percent under a shard that has been serving for an hour. Watchers are keyed by source rather than by shard, because a fetch is single-flight: eight shards starting together produce one download and all eight are waiting on it. No percentage is invented when the server sent no length. One that stops moving is worse than a byte count that keeps rising, so the field is absent and the listing says "so far". Co-Authored-By: Claude Opus 5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 186 ++++++++++++++++------ api/proto/llmgw/v1/agent.proto | 19 +++ internal/agent/agent.go | 84 ++++++++-- internal/agent/agent_test.go | 20 +++ internal/cli/ctl.go | 10 ++ internal/cli/ctlweights.go | 21 +++ internal/control/adminapi/api.go | 29 +++- internal/control/agentsvc/service.go | 12 ++ internal/control/reconciler/fetch_test.go | 70 ++++++++ internal/control/reconciler/reconciler.go | 27 ++++ internal/control/reconciler/replica.go | 29 ++++ 11 files changed, 447 insertions(+), 60 deletions(-) create mode 100644 internal/control/reconciler/fetch_test.go diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 45bafdf..86bf11d 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1134,7 +1134,15 @@ type ShardStatus struct { ExitCode int32 `protobuf:"varint,6,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` // last_output carries the tail of the engine log on a failure. When a model // fails to load this is the only diagnosis available. - LastOutput []string `protobuf:"bytes,7,rep,name=last_output,json=lastOutput,proto3" json:"last_output,omitempty"` + LastOutput []string `protobuf:"bytes,7,rep,name=last_output,json=lastOutput,proto3" json:"last_output,omitempty"` + // fetch is how far a download has got, when the shard is waiting on one. + // + // The agent logs this already, on the node, where nobody is looking. From + // the control plane a replica pulling a hundred gigabytes and one that is + // wedged are the same thing — a shard that has been starting for ten + // minutes — and telling them apart is the difference between waiting and + // going to look. + Fetch *FetchProgress `protobuf:"bytes,8,opt,name=fetch,proto3" json:"fetch,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1218,6 +1226,77 @@ func (x *ShardStatus) GetLastOutput() []string { return nil } +func (x *ShardStatus) GetFetch() *FetchProgress { + if x != nil { + return x.Fetch + } + return nil +} + +// FetchProgress is a download in flight. +type FetchProgress struct { + state protoimpl.MessageState `protogen:"open.v1"` + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + BytesDone int64 `protobuf:"varint,2,opt,name=bytes_done,json=bytesDone,proto3" json:"bytes_done,omitempty"` + // bytes_total is 0 when the server did not say how large the file is, which + // is a real answer: a percentage would then be invented, and an invented + // percentage that stops moving is worse than a byte count that keeps rising. + BytesTotal int64 `protobuf:"varint,3,opt,name=bytes_total,json=bytesTotal,proto3" json:"bytes_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchProgress) Reset() { + *x = FetchProgress{} + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchProgress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchProgress) ProtoMessage() {} + +func (x *FetchProgress) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. +func (*FetchProgress) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *FetchProgress) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *FetchProgress) GetBytesDone() int64 { + if x != nil { + return x.BytesDone + } + return 0 +} + +func (x *FetchProgress) GetBytesTotal() int64 { + if x != nil { + return x.BytesTotal + } + return 0 +} + // StartShard launches one shard. It is idempotent by shard_id: the control // plane retries, and a repeat must not produce a second process. type StartShard struct { @@ -1247,7 +1326,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1338,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1351,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *StartShard) GetReplicaId() string { @@ -1392,7 +1471,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1404,7 +1483,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1417,7 +1496,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *EngineProfile) GetName() string { @@ -1513,7 +1592,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1525,7 +1604,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1538,7 +1617,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *RemoveWeights) GetPaths() []string { @@ -1558,7 +1637,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1570,7 +1649,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1583,7 +1662,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *StopShard) GetShardId() string { @@ -1603,7 +1682,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1615,7 +1694,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1628,7 +1707,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *DrainShard) GetShardId() string { @@ -1650,7 +1729,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1662,7 +1741,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1675,7 +1754,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *Ack) GetMessageId() string { @@ -1797,7 +1876,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\fnew_problems\x18\x03 \x03(\tR\vnewProblems\x12+\n" + "\astorage\x18\x04 \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12'\n" + "\x0fweights_changed\x18\x05 \x01(\bR\x0eweightsChanged\x120\n" + - "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\x8c\x03\n" + + "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\xbb\x03\n" + "\vShardStatus\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" + "\n" + @@ -1807,7 +1886,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05error\x18\x05 \x01(\tR\x05error\x12\x1b\n" + "\texit_code\x18\x06 \x01(\x05R\bexitCode\x12\x1f\n" + "\vlast_output\x18\a \x03(\tR\n" + - "lastOutput\"\xa2\x01\n" + + "lastOutput\x12-\n" + + "\x05fetch\x18\b \x01(\v2\x17.llmgw.v1.FetchProgressR\x05fetch\"\xa2\x01\n" + "\x05State\x12\x15\n" + "\x11STATE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rSTATE_PENDING\x10\x01\x12\x12\n" + @@ -1816,7 +1896,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vSTATE_READY\x10\x04\x12\x12\n" + "\x0eSTATE_DRAINING\x10\x05\x12\x11\n" + "\rSTATE_STOPPED\x10\x06\x12\x10\n" + - "\fSTATE_FAILED\x10\a\"\xba\x03\n" + + "\fSTATE_FAILED\x10\a\"g\n" + + "\rFetchProgress\x12\x16\n" + + "\x06source\x18\x01 \x01(\tR\x06source\x12\x1d\n" + + "\n" + + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + + "\vbytes_total\x18\x03 \x01(\x03R\n" + + "bytesTotal\"\xba\x03\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -1885,7 +1971,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 19) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -1898,29 +1984,30 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus - (*StartShard)(nil), // 11: llmgw.v1.StartShard - (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 13: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 14: llmgw.v1.StopShard - (*DrainShard)(nil), // 15: llmgw.v1.DrainShard - (*Ack)(nil), // 16: llmgw.v1.Ack - nil, // 17: llmgw.v1.Register.LabelsEntry - nil, // 18: llmgw.v1.Register.EngineVersionsEntry - nil, // 19: llmgw.v1.EngineProfile.EnvEntry + (*FetchProgress)(nil), // 11: llmgw.v1.FetchProgress + (*StartShard)(nil), // 12: llmgw.v1.StartShard + (*EngineProfile)(nil), // 13: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 14: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 15: llmgw.v1.StopShard + (*DrainShard)(nil), // 16: llmgw.v1.DrainShard + (*Ack)(nil), // 17: llmgw.v1.Ack + nil, // 18: llmgw.v1.Register.LabelsEntry + nil, // 19: llmgw.v1.Register.EngineVersionsEntry + nil, // 20: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 14, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 15, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 16, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 13, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 17, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 12, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 15, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 16, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 17, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 14, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 18, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 6, // 9: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 7, // 10: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 18, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 19, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 5, // 12: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage @@ -1928,15 +2015,16 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights 0, // 18: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 12, // 19: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 19, // 20: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 21: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 22: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 22, // [22:23] is the sub-list for method output_type - 21, // [21:22] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 11, // 19: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 13, // 20: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 20, // 21: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 22: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 23: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 23, // [23:24] is the sub-list for method output_type + 22, // [22:23] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1962,7 +2050,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 19, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index ecf97f6..9bdedc1 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -229,6 +229,15 @@ message ShardStatus { // fails to load this is the only diagnosis available. repeated string last_output = 7; + // fetch is how far a download has got, when the shard is waiting on one. + // + // The agent logs this already, on the node, where nobody is looking. From + // the control plane a replica pulling a hundred gigabytes and one that is + // wedged are the same thing — a shard that has been starting for ten + // minutes — and telling them apart is the difference between waiting and + // going to look. + FetchProgress fetch = 8; + enum State { STATE_UNSPECIFIED = 0; STATE_PENDING = 1; @@ -241,6 +250,16 @@ message ShardStatus { } } +// FetchProgress is a download in flight. +message FetchProgress { + string source = 1; + int64 bytes_done = 2; + // bytes_total is 0 when the server did not say how large the file is, which + // is a real answer: a percentage would then be invented, and an invented + // percentage that stops moving is worse than a byte count that keeps rising. + int64 bytes_total = 3; +} + // StartShard launches one shard. It is idempotent by shard_id: the control // plane retries, and a repeat must not produce a second process. message StartShard { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e094687..4d2e8c8 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -15,6 +15,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -219,6 +220,12 @@ type Agent struct { // heartbeat carries the list only when it changed. Touched only by the // heartbeat goroutine, which is the only thing that reads or writes it. lastWeights string + + // fetchWatchers are the shards waiting on each source, so progress on one + // download reaches every shard that asked for it. A fetch is single-flight + // — eight shards starting together produce one download — and all eight + // are waiting, so all eight should say so. + fetchWatchers map[string][]func(weights.Progress) } // connectHTTPClient is the HTTP client connect uses; an interface so tests can @@ -456,12 +463,20 @@ func (a *Agent) weights() *weights.Fetcher { CacheDir: cache, Log: a.log, OnProgress: func(p weights.Progress) { - // Logged rather than silent: a replica that is downloading a - // hundred gigabytes and one that is wedged look identical from the - // control plane, and the difference matters to whoever is waiting. a.log.Info("fetching weights", "source", p.Source, "percent", p.Percent(), "bytes", p.BytesDone, "total", p.BytesTotal) + // And upward, to whoever is watching from the control plane. The + // log line above is on the node, which is where nobody is looking + // when a replica has been starting for ten minutes. + a.mu.Lock() + watchers := slices.Clone(a.fetchWatchers[p.Source]) + a.mu.Unlock() + for _, w := range watchers { + if w != nil { + w(p) + } + } }, } return a.fetcher @@ -763,6 +778,42 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha // // It runs in its own goroutine because model loading takes minutes, and the // stream has to keep carrying heartbeats and other commands meanwhile. +// watchFetch registers interest in a source's download progress and returns +// the function that stops watching. +// +// Keyed by source rather than by shard because a fetch is single-flight: eight +// shards starting together produce one download, and every one of them is +// waiting on it. +func (a *Agent) watchFetch(source string, fn func(weights.Progress)) (stop func()) { + a.mu.Lock() + if a.fetchWatchers == nil { + a.fetchWatchers = map[string][]func(weights.Progress){} + } + a.fetchWatchers[source] = append(a.fetchWatchers[source], fn) + i := len(a.fetchWatchers[source]) - 1 + a.mu.Unlock() + + var once sync.Once + return func() { + once.Do(func() { + a.mu.Lock() + defer a.mu.Unlock() + w := a.fetchWatchers[source] + if i < len(w) { + // Cleared rather than removed, so the indexes other watchers + // hold stay valid. The slice is dropped when the last one goes. + w[i] = nil + } + for _, f := range w { + if f != nil { + return + } + } + delete(a.fetchWatchers, source) + }) + } +} + func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) { shardID, replicaID := m.GetShardId(), m.GetReplicaId() @@ -770,16 +821,18 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< a.shardReplica[shardID] = replicaID a.mu.Unlock() - report := func(st supervisor.State, errMsg string, tail []string) { - msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{ - NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID, - State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail, - }}} + send := func(s *llmgwv1.ShardStatus) { + s.NodeId, s.ReplicaId, s.ShardId = a.cfg.NodeID, replicaID, shardID select { - case out <- msg: + case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: s}}: case <-ctx.Done(): } } + report := func(st supervisor.State, errMsg string, tail []string) { + send(&llmgwv1.ShardStatus{ + State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail, + }) + } spec, timeout := agentsvc.ShardSpecFromStart(m) @@ -808,10 +861,23 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< // arrives minutes into a load, in its own words, on a node nobody is // watching — and because eight shards starting together would otherwise // each fetch the same file. + // + // Progress is reported upward while it runs, so the control plane can tell + // a download from a hang. + stop := a.watchFetch(spec.Profile.ModelPath, func(p weights.Progress) { + send(&llmgwv1.ShardStatus{ + State: agentsvc.StateToWire(supervisor.Starting), + Fetch: &llmgwv1.FetchProgress{ + Source: p.Source, BytesDone: p.BytesDone, BytesTotal: p.BytesTotal, + }, + }) + }) if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil { + stop() report(supervisor.Failed, err.Error(), nil) return } else { + stop() spec.Profile.ModelPath = path } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 9ca4e00..75e910f 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -38,6 +38,7 @@ type statusRecorder struct { calls []adoptCall adopted []reconciler.AdoptedShard unclaim []string // shard ids to report back as unclaimed + fetches []fetchReport } type status struct { @@ -55,6 +56,25 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor. } } +// fetches records download progress, so a test can assert that a fetch is +// visible from the control plane rather than only in the node's log. +func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) { + r.mu.Lock() + defer r.mu.Unlock() + r.fetches = append(r.fetches, fetchReport{replicaID, shardID, source, done, total}) +} + +func (r *statusRecorder) fetchReports() []fetchReport { + r.mu.Lock() + defer r.mu.Unlock() + return append([]fetchReport(nil), r.fetches...) +} + +type fetchReport struct { + replicaID, shardID, source string + done, total int64 +} + func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index d76e9ba..918989a 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -335,6 +335,16 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { r.ID, r.Model, r.State, r.Pool, join(nodes), (time.Duration(r.AgeSec) * time.Second).String(), idleColumn(r.IdleSec), r.Endpoint) + // A line rather than a column, because most replicas are not + // downloading anything and a column would be empty for all of + // them. When one is, it is the only thing worth reading. + for _, s := range r.Shards { + if s.Fetch == nil { + continue + } + fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\n", + s.Node, fetchColumn(s.Fetch)) + } } _ = w.Flush() diff --git a/internal/cli/ctlweights.go b/internal/cli/ctlweights.go index 786e124..54e2320 100644 --- a/internal/cli/ctlweights.go +++ b/internal/cli/ctlweights.go @@ -3,6 +3,7 @@ package cli import ( "encoding/json" "fmt" + "strings" "github.com/spf13/cobra" @@ -151,3 +152,23 @@ func fileSize(b int64) string { return "size unknown" } } + +// fetchColumn renders a download's progress. +// +// A percentage when the size is known and a byte count when it is not, because +// a server that does not send Content-Length is common enough — and an +// invented percentage that stops moving is worse than a number that rises. +func fetchColumn(f *adminapi.FetchView) string { + if f.Percent != nil { + return fmt.Sprintf("%s %d%% of %s", shortSource(f.Source), *f.Percent, fileSize(f.BytesTotal)) + } + return fmt.Sprintf("%s %s so far", shortSource(f.Source), fileSize(f.BytesDone)) +} + +// shortSource keeps the end of a source, which is the part that identifies it. +func shortSource(s string) string { + if i := strings.LastIndex(s, "/"); i >= 0 && i < len(s)-1 { + return s[i+1:] + } + return s +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 6024ad5..cae6aa4 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -131,6 +131,22 @@ type ShardView struct { GPUs []int `json:"gpus"` State string `json:"state"` Err string `json:"error,omitempty"` + // Fetch is the download this shard is waiting on, absent when it is not + // waiting on one. It is what tells a replica pulling a hundred gigabytes + // apart from one that is wedged — which look identical without it. + Fetch *FetchView `json:"fetch,omitempty"` +} + +// FetchView is a download in flight. +type FetchView struct { + Source string `json:"source"` + BytesDone int64 `json:"bytes_done"` + // BytesTotal is 0 when the server did not say how large the file is. + BytesTotal int64 `json:"bytes_total,omitempty"` + // Percent is absent rather than zero when the total is unknown: an + // invented percentage that stops moving is worse than a byte count that + // keeps rising. + Percent *int `json:"percent,omitempty"` } // ExplainView is why a model could not be placed. @@ -262,10 +278,19 @@ func (a *API) Replicas(model string) []ReplicaView { v.IdleSec = int64(time.Since(rep.LastUsedAt).Seconds()) } for _, s := range rep.Shards { - v.Shards = append(v.Shards, ShardView{ + sv := ShardView{ ID: s.ID, Node: s.NodeID, Rank: s.Rank, GPUs: s.GPUIndices, State: s.State.String(), Err: s.Err, - }) + } + if f := s.Fetch; f != nil { + sv.Fetch = &FetchView{ + Source: f.Source, BytesDone: f.BytesDone, BytesTotal: f.BytesTotal, + } + if p, ok := f.Percent(); ok { + sv.Fetch.Percent = &p + } + } + v.Shards = append(v.Shards, sv) } out = append(out, v) } diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 0b937b6..1507899 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -24,6 +24,10 @@ import ( // agents report they are already running. The reconciler implements it. type StatusSink interface { OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string) + // OnFetchProgress is separate because it is not a transition: the shard is + // still starting, and treating each report as a state change would reset + // the timers that decide when a slow start has become a failed one. + OnFetchProgress(replicaID, shardID, source string, done, total int64) Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard) } @@ -233,6 +237,14 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) { s.applyHeartbeat(nodeID, m.Heartbeat) case *llmgwv1.AgentMessage_Shard: + // Progress rather than a transition. Falling through to OnShardStatus + // would re-mark the shard as starting on every chunk, which is how a + // download that takes an hour would keep a start timer at zero. + if f := m.Shard.GetFetch(); f != nil { + s.status.OnFetchProgress(m.Shard.GetReplicaId(), m.Shard.GetShardId(), + f.GetSource(), f.GetBytesDone(), f.GetBytesTotal()) + return + } st := StateFromWire(m.Shard.GetState()) s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError()) if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED { diff --git a/internal/control/reconciler/fetch_test.go b/internal/control/reconciler/fetch_test.go new file mode 100644 index 0000000..e9da8b9 --- /dev/null +++ b/internal/control/reconciler/fetch_test.go @@ -0,0 +1,70 @@ +package reconciler + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" +) + +func TestProgressIsVisibleWithoutBeingATransition(t *testing.T) { + // The failure this replaces: a replica pulling a hundred gigabytes and one + // that is wedged look identical from the control plane — both are a shard + // that has been starting for ten minutes. + // + // And progress must not count as a state change. Treating each chunk as + // one would reset the timer that decides when a slow start has become a + // failed one, so a download would keep a replica alive forever. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.Reconcile(t.Context()) + + reps := h.r.Replicas() + if len(reps) != 1 || len(reps[0].Shards) == 0 { + t.Fatalf("expected a placed replica with shards, got %+v", reps) + } + rep, shard := reps[0].ID, reps[0].Shards[0].ID + + h.r.OnFetchProgress(rep, shard, "hf:org/repo/model.gguf", 3<<30, 12<<30) + + got := h.r.Replicas()[0].Shards[0] + if got.Fetch == nil { + t.Fatal("the download is not visible on the shard") + } + if got.Fetch.BytesDone != 3<<30 || got.Fetch.BytesTotal != 12<<30 { + t.Errorf("progress came back as %+v", got.Fetch) + } + if p, ok := got.Fetch.Percent(); !ok || p != 25 { + t.Errorf("percent = %d (known %v), want 25", p, ok) + } +} + +func TestAnUnknownTotalHasNoPercentage(t *testing.T) { + // A server that sends no Content-Length is common. An invented percentage + // that stops moving is worse than a byte count that keeps rising. + f := FetchProgress{Source: "https://x/y.gguf", BytesDone: 500} + if _, ok := f.Percent(); ok { + t.Error("a percentage was invented from an unknown total") + } +} + +func TestATransitionClearsTheDownload(t *testing.T) { + // Left in place it would sit in the listing at ninety-nine percent under + // a shard that has been serving for an hour. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.Reconcile(t.Context()) + reps := h.r.Replicas() + rep, shard := reps[0].ID, reps[0].Shards[0].ID + + h.r.OnFetchProgress(rep, shard, "hf:org/repo/model.gguf", 1, 2) + h.r.OnShardStatus(rep, shard, supervisor.Ready, "") + + if got := h.r.Replicas()[0].Shards[0]; got.Fetch != nil { + t.Errorf("a finished shard still shows a download: %+v", got.Fetch) + } +} + +func TestProgressForAReplicaThatIsGoneIsIgnored(t *testing.T) { + // A download that finishes after its replica was torn down should not + // resurrect anything or panic. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.OnFetchProgress("no-such-replica", "no-such-shard", "x", 1, 2) +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index d870a5f..3da64c1 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -1235,6 +1235,29 @@ type AdoptedShard struct { Endpoint string } +// OnFetchProgress records how far a shard's download has got. +// +// Separate from OnShardStatus because it is not a transition: the shard is +// still starting, and treating each progress report as a state change would +// reset the timers that decide when a slow start becomes a failed one. +func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) { + r.mu.Lock() + defer r.mu.Unlock() + + rep, ok := r.replicas[replicaID] + if !ok { + return + } + for i := range rep.Shards { + if rep.Shards[i].ID != shardID { + continue + } + rep.Shards[i].Fetch = &FetchProgress{ + Source: source, BytesDone: done, BytesTotal: total, + } + } +} + // OnShardStatus records a status report from an agent. func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) { r.markShard(replicaID, shardID, st, errMsg) @@ -1257,6 +1280,10 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e if errMsg != "" { rep.Shards[i].Err = errMsg } + // A real transition means the download is over, one way or the other. + // Left in place it would sit in the listing at ninety-nine percent + // under a shard that has been serving for an hour. + rep.Shards[i].Fetch = nil break } diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index 746e131..eb7be1c 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -65,6 +65,35 @@ type Shard struct { HTTPPort int State supervisor.State Err string + + // Fetch is the download this shard is waiting on, if any. Cleared when the + // shard reaches any other state, so a finished download does not sit in + // the listing looking like one still running. + Fetch *FetchProgress +} + +// FetchProgress is how far a shard's weights have got. +// +// Carried so an operator can tell a replica pulling a hundred gigabytes from +// one that is wedged. From outside they are the same thing — a shard that has +// been starting for ten minutes — and the difference is whether to wait or to +// go and look. +type FetchProgress struct { + Source string + BytesDone int64 + // BytesTotal is 0 when the server did not say. A percentage would then be + // invented, and an invented one that stops moving is worse than a byte + // count that keeps rising. + BytesTotal int64 +} + +// Percent returns progress as a percentage, and false when the total is +// unknown. +func (f FetchProgress) Percent() (int, bool) { + if f.BytesTotal <= 0 { + return 0, false + } + return int(f.BytesDone * 100 / f.BytesTotal), true } // Leader reports whether this shard serves HTTP for the replica.