diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 45bafdf..86bf11d 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1134,7 +1134,15 @@ type ShardStatus struct { ExitCode int32 `protobuf:"varint,6,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` // last_output carries the tail of the engine log on a failure. When a model // fails to load this is the only diagnosis available. - LastOutput []string `protobuf:"bytes,7,rep,name=last_output,json=lastOutput,proto3" json:"last_output,omitempty"` + LastOutput []string `protobuf:"bytes,7,rep,name=last_output,json=lastOutput,proto3" json:"last_output,omitempty"` + // fetch is how far a download has got, when the shard is waiting on one. + // + // The agent logs this already, on the node, where nobody is looking. From + // the control plane a replica pulling a hundred gigabytes and one that is + // wedged are the same thing — a shard that has been starting for ten + // minutes — and telling them apart is the difference between waiting and + // going to look. + Fetch *FetchProgress `protobuf:"bytes,8,opt,name=fetch,proto3" json:"fetch,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1218,6 +1226,77 @@ func (x *ShardStatus) GetLastOutput() []string { return nil } +func (x *ShardStatus) GetFetch() *FetchProgress { + if x != nil { + return x.Fetch + } + return nil +} + +// FetchProgress is a download in flight. +type FetchProgress struct { + state protoimpl.MessageState `protogen:"open.v1"` + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + BytesDone int64 `protobuf:"varint,2,opt,name=bytes_done,json=bytesDone,proto3" json:"bytes_done,omitempty"` + // bytes_total is 0 when the server did not say how large the file is, which + // is a real answer: a percentage would then be invented, and an invented + // percentage that stops moving is worse than a byte count that keeps rising. + BytesTotal int64 `protobuf:"varint,3,opt,name=bytes_total,json=bytesTotal,proto3" json:"bytes_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchProgress) Reset() { + *x = FetchProgress{} + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchProgress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchProgress) ProtoMessage() {} + +func (x *FetchProgress) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. +func (*FetchProgress) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *FetchProgress) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *FetchProgress) GetBytesDone() int64 { + if x != nil { + return x.BytesDone + } + return 0 +} + +func (x *FetchProgress) GetBytesTotal() int64 { + if x != nil { + return x.BytesTotal + } + return 0 +} + // StartShard launches one shard. It is idempotent by shard_id: the control // plane retries, and a repeat must not produce a second process. type StartShard struct { @@ -1247,7 +1326,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1338,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1351,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *StartShard) GetReplicaId() string { @@ -1392,7 +1471,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1404,7 +1483,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1417,7 +1496,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *EngineProfile) GetName() string { @@ -1513,7 +1592,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1525,7 +1604,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1538,7 +1617,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *RemoveWeights) GetPaths() []string { @@ -1558,7 +1637,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1570,7 +1649,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1583,7 +1662,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *StopShard) GetShardId() string { @@ -1603,7 +1682,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1615,7 +1694,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1628,7 +1707,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *DrainShard) GetShardId() string { @@ -1650,7 +1729,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1662,7 +1741,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1675,7 +1754,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *Ack) GetMessageId() string { @@ -1797,7 +1876,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\fnew_problems\x18\x03 \x03(\tR\vnewProblems\x12+\n" + "\astorage\x18\x04 \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12'\n" + "\x0fweights_changed\x18\x05 \x01(\bR\x0eweightsChanged\x120\n" + - "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\x8c\x03\n" + + "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\xbb\x03\n" + "\vShardStatus\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" + "\n" + @@ -1807,7 +1886,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05error\x18\x05 \x01(\tR\x05error\x12\x1b\n" + "\texit_code\x18\x06 \x01(\x05R\bexitCode\x12\x1f\n" + "\vlast_output\x18\a \x03(\tR\n" + - "lastOutput\"\xa2\x01\n" + + "lastOutput\x12-\n" + + "\x05fetch\x18\b \x01(\v2\x17.llmgw.v1.FetchProgressR\x05fetch\"\xa2\x01\n" + "\x05State\x12\x15\n" + "\x11STATE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rSTATE_PENDING\x10\x01\x12\x12\n" + @@ -1816,7 +1896,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vSTATE_READY\x10\x04\x12\x12\n" + "\x0eSTATE_DRAINING\x10\x05\x12\x11\n" + "\rSTATE_STOPPED\x10\x06\x12\x10\n" + - "\fSTATE_FAILED\x10\a\"\xba\x03\n" + + "\fSTATE_FAILED\x10\a\"g\n" + + "\rFetchProgress\x12\x16\n" + + "\x06source\x18\x01 \x01(\tR\x06source\x12\x1d\n" + + "\n" + + "bytes_done\x18\x02 \x01(\x03R\tbytesDone\x12\x1f\n" + + "\vbytes_total\x18\x03 \x01(\x03R\n" + + "bytesTotal\"\xba\x03\n" + "\n" + "StartShard\x12\x1d\n" + "\n" + @@ -1885,7 +1971,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 19) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -1898,29 +1984,30 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus - (*StartShard)(nil), // 11: llmgw.v1.StartShard - (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 13: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 14: llmgw.v1.StopShard - (*DrainShard)(nil), // 15: llmgw.v1.DrainShard - (*Ack)(nil), // 16: llmgw.v1.Ack - nil, // 17: llmgw.v1.Register.LabelsEntry - nil, // 18: llmgw.v1.Register.EngineVersionsEntry - nil, // 19: llmgw.v1.EngineProfile.EnvEntry + (*FetchProgress)(nil), // 11: llmgw.v1.FetchProgress + (*StartShard)(nil), // 12: llmgw.v1.StartShard + (*EngineProfile)(nil), // 13: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 14: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 15: llmgw.v1.StopShard + (*DrainShard)(nil), // 16: llmgw.v1.DrainShard + (*Ack)(nil), // 17: llmgw.v1.Ack + nil, // 18: llmgw.v1.Register.LabelsEntry + nil, // 19: llmgw.v1.Register.EngineVersionsEntry + nil, // 20: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 14, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 15, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 16, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 13, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 17, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 12, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 15, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 16, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 17, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 14, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 18, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 6, // 9: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 7, // 10: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 18, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 19, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 5, // 12: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage @@ -1928,15 +2015,16 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights 0, // 18: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 12, // 19: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 19, // 20: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 21: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 22: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 22, // [22:23] is the sub-list for method output_type - 21, // [21:22] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 11, // 19: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 13, // 20: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 20, // 21: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 22: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 23: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 23, // [23:24] is the sub-list for method output_type + 22, // [22:23] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1962,7 +2050,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 19, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index ecf97f6..9bdedc1 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -229,6 +229,15 @@ message ShardStatus { // fails to load this is the only diagnosis available. repeated string last_output = 7; + // fetch is how far a download has got, when the shard is waiting on one. + // + // The agent logs this already, on the node, where nobody is looking. From + // the control plane a replica pulling a hundred gigabytes and one that is + // wedged are the same thing — a shard that has been starting for ten + // minutes — and telling them apart is the difference between waiting and + // going to look. + FetchProgress fetch = 8; + enum State { STATE_UNSPECIFIED = 0; STATE_PENDING = 1; @@ -241,6 +250,16 @@ message ShardStatus { } } +// FetchProgress is a download in flight. +message FetchProgress { + string source = 1; + int64 bytes_done = 2; + // bytes_total is 0 when the server did not say how large the file is, which + // is a real answer: a percentage would then be invented, and an invented + // percentage that stops moving is worse than a byte count that keeps rising. + int64 bytes_total = 3; +} + // StartShard launches one shard. It is idempotent by shard_id: the control // plane retries, and a repeat must not produce a second process. message StartShard { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e094687..4d2e8c8 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -15,6 +15,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -219,6 +220,12 @@ type Agent struct { // heartbeat carries the list only when it changed. Touched only by the // heartbeat goroutine, which is the only thing that reads or writes it. lastWeights string + + // fetchWatchers are the shards waiting on each source, so progress on one + // download reaches every shard that asked for it. A fetch is single-flight + // — eight shards starting together produce one download — and all eight + // are waiting, so all eight should say so. + fetchWatchers map[string][]func(weights.Progress) } // connectHTTPClient is the HTTP client connect uses; an interface so tests can @@ -456,12 +463,20 @@ func (a *Agent) weights() *weights.Fetcher { CacheDir: cache, Log: a.log, OnProgress: func(p weights.Progress) { - // Logged rather than silent: a replica that is downloading a - // hundred gigabytes and one that is wedged look identical from the - // control plane, and the difference matters to whoever is waiting. a.log.Info("fetching weights", "source", p.Source, "percent", p.Percent(), "bytes", p.BytesDone, "total", p.BytesTotal) + // And upward, to whoever is watching from the control plane. The + // log line above is on the node, which is where nobody is looking + // when a replica has been starting for ten minutes. + a.mu.Lock() + watchers := slices.Clone(a.fetchWatchers[p.Source]) + a.mu.Unlock() + for _, w := range watchers { + if w != nil { + w(p) + } + } }, } return a.fetcher @@ -763,6 +778,42 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha // // It runs in its own goroutine because model loading takes minutes, and the // stream has to keep carrying heartbeats and other commands meanwhile. +// watchFetch registers interest in a source's download progress and returns +// the function that stops watching. +// +// Keyed by source rather than by shard because a fetch is single-flight: eight +// shards starting together produce one download, and every one of them is +// waiting on it. +func (a *Agent) watchFetch(source string, fn func(weights.Progress)) (stop func()) { + a.mu.Lock() + if a.fetchWatchers == nil { + a.fetchWatchers = map[string][]func(weights.Progress){} + } + a.fetchWatchers[source] = append(a.fetchWatchers[source], fn) + i := len(a.fetchWatchers[source]) - 1 + a.mu.Unlock() + + var once sync.Once + return func() { + once.Do(func() { + a.mu.Lock() + defer a.mu.Unlock() + w := a.fetchWatchers[source] + if i < len(w) { + // Cleared rather than removed, so the indexes other watchers + // hold stay valid. The slice is dropped when the last one goes. + w[i] = nil + } + for _, f := range w { + if f != nil { + return + } + } + delete(a.fetchWatchers, source) + }) + } +} + func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) { shardID, replicaID := m.GetShardId(), m.GetReplicaId() @@ -770,16 +821,18 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< a.shardReplica[shardID] = replicaID a.mu.Unlock() - report := func(st supervisor.State, errMsg string, tail []string) { - msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{ - NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID, - State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail, - }}} + send := func(s *llmgwv1.ShardStatus) { + s.NodeId, s.ReplicaId, s.ShardId = a.cfg.NodeID, replicaID, shardID select { - case out <- msg: + case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: s}}: case <-ctx.Done(): } } + report := func(st supervisor.State, errMsg string, tail []string) { + send(&llmgwv1.ShardStatus{ + State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail, + }) + } spec, timeout := agentsvc.ShardSpecFromStart(m) @@ -808,10 +861,23 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< // arrives minutes into a load, in its own words, on a node nobody is // watching — and because eight shards starting together would otherwise // each fetch the same file. + // + // Progress is reported upward while it runs, so the control plane can tell + // a download from a hang. + stop := a.watchFetch(spec.Profile.ModelPath, func(p weights.Progress) { + send(&llmgwv1.ShardStatus{ + State: agentsvc.StateToWire(supervisor.Starting), + Fetch: &llmgwv1.FetchProgress{ + Source: p.Source, BytesDone: p.BytesDone, BytesTotal: p.BytesTotal, + }, + }) + }) if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil { + stop() report(supervisor.Failed, err.Error(), nil) return } else { + stop() spec.Profile.ModelPath = path } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 9ca4e00..75e910f 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -38,6 +38,7 @@ type statusRecorder struct { calls []adoptCall adopted []reconciler.AdoptedShard unclaim []string // shard ids to report back as unclaimed + fetches []fetchReport } type status struct { @@ -55,6 +56,25 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor. } } +// fetches records download progress, so a test can assert that a fetch is +// visible from the control plane rather than only in the node's log. +func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) { + r.mu.Lock() + defer r.mu.Unlock() + r.fetches = append(r.fetches, fetchReport{replicaID, shardID, source, done, total}) +} + +func (r *statusRecorder) fetchReports() []fetchReport { + r.mu.Lock() + defer r.mu.Unlock() + return append([]fetchReport(nil), r.fetches...) +} + +type fetchReport struct { + replicaID, shardID, source string + done, total int64 +} + func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index d76e9ba..918989a 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -335,6 +335,16 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { r.ID, r.Model, r.State, r.Pool, join(nodes), (time.Duration(r.AgeSec) * time.Second).String(), idleColumn(r.IdleSec), r.Endpoint) + // A line rather than a column, because most replicas are not + // downloading anything and a column would be empty for all of + // them. When one is, it is the only thing worth reading. + for _, s := range r.Shards { + if s.Fetch == nil { + continue + } + fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\n", + s.Node, fetchColumn(s.Fetch)) + } } _ = w.Flush() diff --git a/internal/cli/ctlweights.go b/internal/cli/ctlweights.go index 786e124..54e2320 100644 --- a/internal/cli/ctlweights.go +++ b/internal/cli/ctlweights.go @@ -3,6 +3,7 @@ package cli import ( "encoding/json" "fmt" + "strings" "github.com/spf13/cobra" @@ -151,3 +152,23 @@ func fileSize(b int64) string { return "size unknown" } } + +// fetchColumn renders a download's progress. +// +// A percentage when the size is known and a byte count when it is not, because +// a server that does not send Content-Length is common enough — and an +// invented percentage that stops moving is worse than a number that rises. +func fetchColumn(f *adminapi.FetchView) string { + if f.Percent != nil { + return fmt.Sprintf("%s %d%% of %s", shortSource(f.Source), *f.Percent, fileSize(f.BytesTotal)) + } + return fmt.Sprintf("%s %s so far", shortSource(f.Source), fileSize(f.BytesDone)) +} + +// shortSource keeps the end of a source, which is the part that identifies it. +func shortSource(s string) string { + if i := strings.LastIndex(s, "/"); i >= 0 && i < len(s)-1 { + return s[i+1:] + } + return s +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 6024ad5..cae6aa4 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -131,6 +131,22 @@ type ShardView struct { GPUs []int `json:"gpus"` State string `json:"state"` Err string `json:"error,omitempty"` + // Fetch is the download this shard is waiting on, absent when it is not + // waiting on one. It is what tells a replica pulling a hundred gigabytes + // apart from one that is wedged — which look identical without it. + Fetch *FetchView `json:"fetch,omitempty"` +} + +// FetchView is a download in flight. +type FetchView struct { + Source string `json:"source"` + BytesDone int64 `json:"bytes_done"` + // BytesTotal is 0 when the server did not say how large the file is. + BytesTotal int64 `json:"bytes_total,omitempty"` + // Percent is absent rather than zero when the total is unknown: an + // invented percentage that stops moving is worse than a byte count that + // keeps rising. + Percent *int `json:"percent,omitempty"` } // ExplainView is why a model could not be placed. @@ -262,10 +278,19 @@ func (a *API) Replicas(model string) []ReplicaView { v.IdleSec = int64(time.Since(rep.LastUsedAt).Seconds()) } for _, s := range rep.Shards { - v.Shards = append(v.Shards, ShardView{ + sv := ShardView{ ID: s.ID, Node: s.NodeID, Rank: s.Rank, GPUs: s.GPUIndices, State: s.State.String(), Err: s.Err, - }) + } + if f := s.Fetch; f != nil { + sv.Fetch = &FetchView{ + Source: f.Source, BytesDone: f.BytesDone, BytesTotal: f.BytesTotal, + } + if p, ok := f.Percent(); ok { + sv.Fetch.Percent = &p + } + } + v.Shards = append(v.Shards, sv) } out = append(out, v) } diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 0b937b6..1507899 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -24,6 +24,10 @@ import ( // agents report they are already running. The reconciler implements it. type StatusSink interface { OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string) + // OnFetchProgress is separate because it is not a transition: the shard is + // still starting, and treating each report as a state change would reset + // the timers that decide when a slow start has become a failed one. + OnFetchProgress(replicaID, shardID, source string, done, total int64) Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard) } @@ -233,6 +237,14 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) { s.applyHeartbeat(nodeID, m.Heartbeat) case *llmgwv1.AgentMessage_Shard: + // Progress rather than a transition. Falling through to OnShardStatus + // would re-mark the shard as starting on every chunk, which is how a + // download that takes an hour would keep a start timer at zero. + if f := m.Shard.GetFetch(); f != nil { + s.status.OnFetchProgress(m.Shard.GetReplicaId(), m.Shard.GetShardId(), + f.GetSource(), f.GetBytesDone(), f.GetBytesTotal()) + return + } st := StateFromWire(m.Shard.GetState()) s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError()) if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED { diff --git a/internal/control/reconciler/fetch_test.go b/internal/control/reconciler/fetch_test.go new file mode 100644 index 0000000..e9da8b9 --- /dev/null +++ b/internal/control/reconciler/fetch_test.go @@ -0,0 +1,70 @@ +package reconciler + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" +) + +func TestProgressIsVisibleWithoutBeingATransition(t *testing.T) { + // The failure this replaces: a replica pulling a hundred gigabytes and one + // that is wedged look identical from the control plane — both are a shard + // that has been starting for ten minutes. + // + // And progress must not count as a state change. Treating each chunk as + // one would reset the timer that decides when a slow start has become a + // failed one, so a download would keep a replica alive forever. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.Reconcile(t.Context()) + + reps := h.r.Replicas() + if len(reps) != 1 || len(reps[0].Shards) == 0 { + t.Fatalf("expected a placed replica with shards, got %+v", reps) + } + rep, shard := reps[0].ID, reps[0].Shards[0].ID + + h.r.OnFetchProgress(rep, shard, "hf:org/repo/model.gguf", 3<<30, 12<<30) + + got := h.r.Replicas()[0].Shards[0] + if got.Fetch == nil { + t.Fatal("the download is not visible on the shard") + } + if got.Fetch.BytesDone != 3<<30 || got.Fetch.BytesTotal != 12<<30 { + t.Errorf("progress came back as %+v", got.Fetch) + } + if p, ok := got.Fetch.Percent(); !ok || p != 25 { + t.Errorf("percent = %d (known %v), want 25", p, ok) + } +} + +func TestAnUnknownTotalHasNoPercentage(t *testing.T) { + // A server that sends no Content-Length is common. An invented percentage + // that stops moving is worse than a byte count that keeps rising. + f := FetchProgress{Source: "https://x/y.gguf", BytesDone: 500} + if _, ok := f.Percent(); ok { + t.Error("a percentage was invented from an unknown total") + } +} + +func TestATransitionClearsTheDownload(t *testing.T) { + // Left in place it would sit in the listing at ninety-nine percent under + // a shard that has been serving for an hour. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.Reconcile(t.Context()) + reps := h.r.Replicas() + rep, shard := reps[0].ID, reps[0].Shards[0].ID + + h.r.OnFetchProgress(rep, shard, "hf:org/repo/model.gguf", 1, 2) + h.r.OnShardStatus(rep, shard, supervisor.Ready, "") + + if got := h.r.Replicas()[0].Shards[0]; got.Fetch != nil { + t.Errorf("a finished shard still shows a download: %+v", got.Fetch) + } +} + +func TestProgressForAReplicaThatIsGoneIsIgnored(t *testing.T) { + // A download that finishes after its replica was torn down should not + // resurrect anything or panic. + h := newHarness(t, testModel(1), gpuNode("n1", 8)) + h.r.OnFetchProgress("no-such-replica", "no-such-shard", "x", 1, 2) +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index d870a5f..3da64c1 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -1235,6 +1235,29 @@ type AdoptedShard struct { Endpoint string } +// OnFetchProgress records how far a shard's download has got. +// +// Separate from OnShardStatus because it is not a transition: the shard is +// still starting, and treating each progress report as a state change would +// reset the timers that decide when a slow start becomes a failed one. +func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) { + r.mu.Lock() + defer r.mu.Unlock() + + rep, ok := r.replicas[replicaID] + if !ok { + return + } + for i := range rep.Shards { + if rep.Shards[i].ID != shardID { + continue + } + rep.Shards[i].Fetch = &FetchProgress{ + Source: source, BytesDone: done, BytesTotal: total, + } + } +} + // OnShardStatus records a status report from an agent. func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) { r.markShard(replicaID, shardID, st, errMsg) @@ -1257,6 +1280,10 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e if errMsg != "" { rep.Shards[i].Err = errMsg } + // A real transition means the download is over, one way or the other. + // Left in place it would sit in the listing at ninety-nine percent + // under a shard that has been serving for an hour. + rep.Shards[i].Fetch = nil break } diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index 746e131..eb7be1c 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -65,6 +65,35 @@ type Shard struct { HTTPPort int State supervisor.State Err string + + // Fetch is the download this shard is waiting on, if any. Cleared when the + // shard reaches any other state, so a finished download does not sit in + // the listing looking like one still running. + Fetch *FetchProgress +} + +// FetchProgress is how far a shard's weights have got. +// +// Carried so an operator can tell a replica pulling a hundred gigabytes from +// one that is wedged. From outside they are the same thing — a shard that has +// been starting for ten minutes — and the difference is whether to wait or to +// go and look. +type FetchProgress struct { + Source string + BytesDone int64 + // BytesTotal is 0 when the server did not say. A percentage would then be + // invented, and an invented one that stops moving is worse than a byte + // count that keeps rising. + BytesTotal int64 +} + +// Percent returns progress as a percentage, and false when the total is +// unknown. +func (f FetchProgress) Percent() (int, bool) { + if f.BytesTotal <= 0 { + return 0, false + } + return int(f.BytesDone * 100 / f.BytesTotal), true } // Leader reports whether this shard serves HTTP for the replica.