diff --git a/CHANGELOG.md b/CHANGELOG.md index 9344dec..1927185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,17 @@ more than 26 hours old or there has never been one. docs/install.md now says how to restore, including why running engines survive it: the engine-key secret comes back with the store. +### Upgrading from one command + +`sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that +machine and then every node in turn. Each agent is told the version over its +existing stream, fetches the release for its own platform, checks it against +the release's checksums, keeps the old binary beside the new and restarts, +while its engines keep serving. A node that does not come back on the new +version, ready, with its replicas, stops the upgrade before the next one. +Running it again resumes. Agents from before this have to be moved by hand +once. + ### Following a request Every response carries `X-Request-Id` — the client's own when it sends a plain diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 57a6fff..b0ef34a 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -104,6 +104,7 @@ type AgentMessage struct { // *AgentMessage_Logs // *AgentMessage_FabricScan // *AgentMessage_FabricTest + // *AgentMessage_Upgrade Msg isAgentMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -200,6 +201,15 @@ func (x *AgentMessage) GetFabricTest() *FabricTestReply { return nil } +func (x *AgentMessage) GetUpgrade() *UpgradeReply { + if x != nil { + if x, ok := x.Msg.(*AgentMessage_Upgrade); ok { + return x.Upgrade + } + } + return nil +} + type isAgentMessage_Msg interface { isAgentMessage_Msg() } @@ -228,6 +238,10 @@ type AgentMessage_FabricTest struct { FabricTest *FabricTestReply `protobuf:"bytes,6,opt,name=fabric_test,json=fabricTest,proto3,oneof"` } +type AgentMessage_Upgrade struct { + Upgrade *UpgradeReply `protobuf:"bytes,7,opt,name=upgrade,proto3,oneof"` +} + func (*AgentMessage_Register) isAgentMessage_Msg() {} func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {} @@ -240,6 +254,8 @@ func (*AgentMessage_FabricScan) isAgentMessage_Msg() {} func (*AgentMessage_FabricTest) isAgentMessage_Msg() {} +func (*AgentMessage_Upgrade) isAgentMessage_Msg() {} + // ControlMessage is anything the control plane tells a node. type ControlMessage struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -254,6 +270,7 @@ type ControlMessage struct { // *ControlMessage_Logs // *ControlMessage_FabricScan // *ControlMessage_FabricTest + // *ControlMessage_Upgrade Msg isControlMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -377,6 +394,15 @@ func (x *ControlMessage) GetFabricTest() *FabricTestRequest { return nil } +func (x *ControlMessage) GetUpgrade() *UpgradeAgent { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_Upgrade); ok { + return x.Upgrade + } + } + return nil +} + type isControlMessage_Msg interface { isControlMessage_Msg() } @@ -417,6 +443,10 @@ type ControlMessage_FabricTest struct { FabricTest *FabricTestRequest `protobuf:"bytes,9,opt,name=fabric_test,json=fabricTest,proto3,oneof"` } +type ControlMessage_Upgrade struct { + Upgrade *UpgradeAgent `protobuf:"bytes,10,opt,name=upgrade,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -435,6 +465,8 @@ func (*ControlMessage_FabricScan) isControlMessage_Msg() {} func (*ControlMessage_FabricTest) isControlMessage_Msg() {} +func (*ControlMessage_Upgrade) isControlMessage_Msg() {} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. @@ -3460,11 +3492,144 @@ func (x *FabricTestReply) GetTookMs() int64 { return 0 } +// UpgradeAgent tells an agent to replace its own binary with a release and +// restart. +// +// The agent fetches the release for its own platform and checks it against +// the release's checksums before replacing anything, keeps the old binary +// beside the new, then exits so systemd starts the new one. Engines keep +// running throughout, as for any agent restart, and are adopted on the way +// back. Over the stream the agent already holds, so upgrading a fleet needs +// no ssh into it. +type UpgradeAgent struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // version must be a release: v0.10.4 or v0.10.4-rc.2. + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + // releases is the base URL release files are downloaded from. + Releases string `protobuf:"bytes,3,opt,name=releases,proto3" json:"releases,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpgradeAgent) Reset() { + *x = UpgradeAgent{} + mi := &file_llmgw_v1_agent_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpgradeAgent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpgradeAgent) ProtoMessage() {} + +func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead. +func (*UpgradeAgent) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} +} + +func (x *UpgradeAgent) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (x *UpgradeAgent) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *UpgradeAgent) GetReleases() string { + if x != nil { + return x.Releases + } + return "" +} + +// UpgradeReply says whether the new binary is in place. Sent before the +// agent exits; an empty error means it will be back on the new version. +type UpgradeReply struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UpgradeReply) Reset() { + *x = UpgradeReply{} + mi := &file_llmgw_v1_agent_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UpgradeReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpgradeReply) ProtoMessage() {} + +func (x *UpgradeReply) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead. +func (*UpgradeReply) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} +} + +func (x *UpgradeReply) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (x *UpgradeReply) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *UpgradeReply) GetError() string { + if x != nil { + return x.Error + } + return "" +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + - "\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd2\x02\n" + + "\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x86\x03\n" + "\fAgentMessage\x120\n" + "\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" + "\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" + @@ -3473,8 +3638,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" + "fabricScan\x12<\n" + "\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" + - "fabricTestB\x05\n" + - "\x03msg\"\xe5\x03\n" + + "fabricTest\x122\n" + + "\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgradeB\x05\n" + + "\x03msg\"\x99\x04\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + @@ -3486,7 +3652,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" + "fabricScan\x12>\n" + "\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" + - "fabricTestB\x05\n" + + "fabricTest\x122\n" + + "\aupgrade\x18\n" + + " \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" + "\x03msg\"\xe3\x06\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + @@ -3795,7 +3963,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04gbps\x18\x02 \x01(\x01R\x04gbps\x12\x16\n" + "\x06output\x18\x03 \x01(\tR\x06output\x12\x14\n" + "\x05error\x18\x04 \x01(\tR\x05error\x12\x17\n" + - "\atook_ms\x18\x05 \x01(\x03R\x06tookMs2O\n" + + "\atook_ms\x18\x05 \x01(\x03R\x06tookMs\"c\n" + + "\fUpgradeAgent\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" + + "\aversion\x18\x02 \x01(\tR\aversion\x12\x1a\n" + + "\breleases\x18\x03 \x01(\tR\breleases\"]\n" + + "\fUpgradeReply\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" + + "\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" + + "\x05error\x18\x03 \x01(\tR\x05error2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + @@ -3814,7 +3992,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 41) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -3850,12 +4028,14 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply (*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest (*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply - nil, // 34: llmgw.v1.Register.LabelsEntry - nil, // 35: llmgw.v1.Register.EngineVersionsEntry - nil, // 36: llmgw.v1.GPU.NicPathsEntry - nil, // 37: llmgw.v1.IBPort.CountersEntry - nil, // 38: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 39: llmgw.v1.EngineProfile.EnvEntry + (*UpgradeAgent)(nil), // 34: llmgw.v1.UpgradeAgent + (*UpgradeReply)(nil), // 35: llmgw.v1.UpgradeReply + nil, // 36: llmgw.v1.Register.LabelsEntry + nil, // 37: llmgw.v1.Register.EngineVersionsEntry + nil, // 38: llmgw.v1.GPU.NicPathsEntry + nil, // 39: llmgw.v1.IBPort.CountersEntry + nil, // 40: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 41: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -3864,50 +4044,52 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply 31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply 33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply - 22, // 6: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 25, // 7: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 26, // 8: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 27, // 9: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 24, // 10: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 17, // 11: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 28, // 12: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 30, // 13: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest - 32, // 14: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest - 34, // 15: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 7, // 16: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 8, // 17: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 35, // 18: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 6, // 19: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 9, // 20: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 5, // 21: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 4, // 22: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 14, // 23: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX - 0, // 24: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 36, // 25: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 11, // 26: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 37, // 27: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 13, // 28: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 5, // 29: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 30: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 38, // 31: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 19, // 32: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 33: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 12, // 34: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 8, // 35: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 15, // 36: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState - 14, // 37: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX - 18, // 38: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 39: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 21, // 40: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 23, // 41: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 39, // 42: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 43: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 44: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 44, // [44:45] is the sub-list for method output_type - 43, // [43:44] is the sub-list for method input_type - 43, // [43:43] is the sub-list for extension type_name - 43, // [43:43] is the sub-list for extension extendee - 0, // [0:43] is the sub-list for field type_name + 35, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply + 22, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 25, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 26, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 27, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 24, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 17, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 28, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 30, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 32, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest + 34, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent + 36, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 37, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host + 14, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX + 0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 38, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 11, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 39, // 29: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 13, // 30: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 5, // 31: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 32: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 40, // 33: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 19, // 34: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 35: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 12, // 36: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 8, // 37: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 15, // 38: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState + 14, // 39: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX + 18, // 40: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 41: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 21, // 42: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 23, // 43: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 41, // 44: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 45: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 46: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 46, // [46:47] is the sub-list for method output_type + 45, // [45:46] is the sub-list for method input_type + 45, // [45:45] is the sub-list for extension type_name + 45, // [45:45] is the sub-list for extension extendee + 0, // [0:45] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -3922,6 +4104,7 @@ func file_llmgw_v1_agent_proto_init() { (*AgentMessage_Logs)(nil), (*AgentMessage_FabricScan)(nil), (*AgentMessage_FabricTest)(nil), + (*AgentMessage_Upgrade)(nil), } file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{ (*ControlMessage_Start)(nil), @@ -3933,6 +4116,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_Logs)(nil), (*ControlMessage_FabricScan)(nil), (*ControlMessage_FabricTest)(nil), + (*ControlMessage_Upgrade)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -3940,7 +4124,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 39, + NumMessages: 41, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 36d620b..811f095 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -26,6 +26,7 @@ message AgentMessage { LogsReply logs = 4; FabricScanReply fabric_scan = 5; FabricTestReply fabric_test = 6; + UpgradeReply upgrade = 7; } } @@ -41,6 +42,7 @@ message ControlMessage { LogsRequest logs = 7; FabricScanRequest fabric_scan = 8; FabricTestRequest fabric_test = 9; + UpgradeAgent upgrade = 10; } } @@ -649,3 +651,28 @@ message FabricTestReply { string error = 4; int64 took_ms = 5; } + +// UpgradeAgent tells an agent to replace its own binary with a release and +// restart. +// +// The agent fetches the release for its own platform and checks it against +// the release's checksums before replacing anything, keeps the old binary +// beside the new, then exits so systemd starts the new one. Engines keep +// running throughout, as for any agent restart, and are adopted on the way +// back. Over the stream the agent already holds, so upgrading a fleet needs +// no ssh into it. +message UpgradeAgent { + string request_id = 1; + // version must be a release: v0.10.4 or v0.10.4-rc.2. + string version = 2; + // releases is the base URL release files are downloaded from. + string releases = 3; +} + +// UpgradeReply says whether the new binary is in place. Sent before the +// agent exits; an empty error means it will be back on the new version. +message UpgradeReply { + string request_id = 1; + string version = 2; + string error = 3; +} diff --git a/docs/install.md b/docs/install.md index 27d5409..f327ecf 100644 --- a/docs/install.md +++ b/docs/install.md @@ -909,55 +909,52 @@ refuses every request. ## Upgrading -One binary on every machine, so an upgrade is a copy and a restart. The part -worth care is the copy, not the order. +On the control plane: + +``` +sudo llmgw ctl upgrade v0.10.4 +``` + +It upgrades this machine first — fetching the release for its platform, +checking it against the release's checksums, keeping the old binary as +`/usr/local/bin/llmgw.bak-pre-`, restarting `llmgw-server` and waiting +for every node to reconnect — and then each node in turn. A node's agent is told +the version over the stream it already holds, fetches and checks the release +itself, swaps it in and restarts; no ssh into the nodes is needed. Release +candidates (`v0.10.4-rc.2`) work the same way. + +A node counts as done when it is back on the new version, ready, with as many +ready replicas as before. One that is not within `--timeout` (three minutes) +stops the upgrade there, so a bad release reaches one node rather than all of +them; `llmgw ctl logs ` says why, and the node keeps its previous binary +beside the new one. Nodes already on the version are skipped, so running the +command again resumes where it stopped. `--nodes a,b` limits it to some nodes, +and `--skip-control-plane` leaves this machine alone. **Nothing stops serving.** Engines are started with `KillMode=process`, so they keep running when the agent restarts, and the agent reports them back on its next connection. The control plane holds no replica state across its own -restart either — it rebuilds the fleet from what the agents report. A rolling -upgrade of the whole cluster interrupts no request that is not in flight at the -moment of the restart. +restart either — it rebuilds the fleet from what the agents report. **A mixed fleet is fine while you roll.** The agent protocol only ever gains fields, and both sides treat an absent one as unknown rather than as a default. -Order does not matter; finish the roll the same day anyway, because -`ctl nodes` reports each agent's version and a fleet that disagrees with itself -is one more thing to hold in your head. - -``` -VERSION=v0.9.0 -# copy to every machine first, install nowhere yet -for h in 10.0.0.140 10.0.0.141 10.0.0.142 10.0.0.146; do - ssh $h 'cat > /tmp/llmgw-new; stat -c %s /tmp/llmgw-new' < llmgw-$VERSION-linux-amd64 -done -``` - -**Check every size before installing anything.** This is not defensive writing: -on a host whose login shell prints anything, `scp` fails outright, and the -obvious workaround — `ssh -n host 'cat > file'` — points stdin at `/dev/null` -and writes a *zero-byte file* while reporting success. Installing that on a -fleet takes the fleet down. Compare the number on each machine with the local -one, then: - -``` -sudo install -m0755 /tmp/llmgw-new /usr/local/bin/llmgw -sudo systemctl restart llmgw-agent # or llmgw-server, or llmgw-mirror -llmgw --version -``` +Agents older than v0.10.4 cannot upgrade themselves: move those by hand once, +installing the release binary and restarting `llmgw-agent`, and every upgrade +after that is the command above. The mirror, which is not an agent, is +upgraded the same way by hand. Then confirm what you have, rather than assuming: ``` -llmgw ctl nodes # every agent back, LAST SEEN in seconds +llmgw ctl nodes # every agent on the new version, LAST SEEN in seconds llmgw ctl replicas # same replicas, adopted rather than replaced ``` A replica may come back marked **`ready (stale profile)`**. That means it is running a profile that has since been edited — the reconciler leaves a serving replica alone on purpose, because churning one costs more than the difference. -Restart it when you want the new profile to take effect; the flag is there so -the choice is yours rather than silent. +`llmgw ctl restart ` replaces it with one on the new profile, starting +the new one before stopping the old. ## Backing up diff --git a/docs/reference.md b/docs/reference.md index 79379ef..d2688ba 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -170,6 +170,7 @@ rather than repeated. | `mirror` | `Mirror` | [Server](#server) | | `engines` | `map[string]string` | [Server](#server) | | `backup_dir` | `string` | [Server](#server) | +| `releases` | `string` | [Server](#server) | | `router` | `Router` | [Server](#server) | | `control` | `Control` | [Server](#server) | | `models` | `[]catalog.Model` | [Server](#server) | @@ -1274,6 +1275,14 @@ BackupDir is where `llmgw server backup` writes, and where the newest archive's age is read for llmgw_backup_last_success_timestamp_seconds. Empty means /var/backups/llmgw-server. +### `releases` + +`string` + +Releases is where `llmgw ctl upgrade` fetches release binaries from, +for this machine and — through their agents — every node. Empty means +the project's own releases. + ### `router` `Router` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 5282735..62e570f 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "errors" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" "io" "log/slog" "maps" @@ -18,6 +19,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -1119,6 +1121,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha a.removeWeights(m.RemoveWeights.GetPaths()) case *llmgwv1.ControlMessage_Provision: a.setProvision(m.Provision) + case *llmgwv1.ControlMessage_Upgrade: + go a.upgrade(ctx, m.Upgrade, out) case *llmgwv1.ControlMessage_Logs: reply := a.logsReply(m.Logs) // Off the receive loop: a full send queue must not stop commands being @@ -1874,3 +1878,49 @@ func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) { m := discoverIMEX(ctx) hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)} } + +// upgradeExit is the exit status after an upgrade: non-zero, so systemd's +// Restart=on-failure starts the new binary, and distinct, so the journal says +// why the agent went. +const upgradeExit = 75 + +// exitForUpgrade ends the process; a variable so a test can see it called +// rather than have its own process end. +var exitForUpgrade = func() { os.Exit(upgradeExit) } + +// upgradeTarget is the binary an upgrade replaces; a variable so a test +// replaces a file of its own rather than the test binary running it. +var upgradeTarget = os.Executable + +// upgrade replaces this agent's binary with a release and restarts into it. +// +// Engines are not touched: an agent restart leaves them running and adopts +// them again, which is what makes upgrading a node something that can be done +// while it serves. +func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) { + reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()} + send := func() { + select { + case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}: + case <-ctx.Done(): + } + } + a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version) + self, err := upgradeTarget() + if err == nil { + dctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + _, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self) + cancel() + } + if err != nil { + reply.Error = err.Error() + a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err) + send() + return + } + send() + a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion()) + // A moment for the reply to leave before the stream goes. + time.Sleep(500 * time.Millisecond) + exitForUpgrade() +} diff --git a/internal/agent/upgrade_test.go b/internal/agent/upgrade_test.go new file mode 100644 index 0000000..5a67824 --- /dev/null +++ b/internal/agent/upgrade_test.go @@ -0,0 +1,78 @@ +package agent + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" +) + +func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) { + body := []byte("the new agent") + sum := sha256.Sum256(body) + asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v9.9.9/checksums.txt": + _, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n")) + case "/v9.9.9/" + asset: + _, _ = w.Write(body) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + bin := filepath.Join(t.TempDir(), "llmgw") + _ = os.WriteFile(bin, []byte("the old agent"), 0o755) + exited := make(chan struct{}, 1) + oldTarget, oldExit := upgradeTarget, exitForUpgrade + upgradeTarget = func() (string, error) { return bin, nil } + exitForUpgrade = func() { exited <- struct{}{} } + t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit }) + + cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()} + cfg.Superv.WorkDir = t.TempDir() + a, err := New(cfg, nil, nil) + if err != nil { + t.Fatal(err) + } + out := make(chan *llmgwv1.AgentMessage, 1) + go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out) + + select { + case m := <-out: + if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" { + t.Fatalf("reply %+v", r) + } + case <-time.After(10 * time.Second): + t.Fatal("no reply") + } + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("the agent did not restart into the new binary") + } + if b, _ := os.ReadFile(bin); string(b) != "the new agent" { + t.Errorf("binary holds %q", b) + } + + // A version with no release answers with why, and does not restart. + go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out) + if r := (<-out).GetUpgrade(); r.GetError() == "" { + t.Error("an upgrade to a release that does not exist reported success") + } + select { + case <-exited: + t.Error("a failed upgrade restarted the agent") + case <-time.After(time.Second): + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 0ce4129..45a6d3e 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -141,6 +141,7 @@ func newCtlCmd() *cobra.Command { newScaleCmd(client), newRestartCmd(client), newRetryCmd(client), + newUpgradeCmd(client), newEngineCheckCmd(), newToolCheckCmd(), newWeightsCmd(client, &asJSON), diff --git a/internal/cli/ctl_upgrade.go b/internal/cli/ctl_upgrade.go new file mode 100644 index 0000000..2a57915 --- /dev/null +++ b/internal/cli/ctl_upgrade.go @@ -0,0 +1,197 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "runtime" + "slices" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +func newUpgradeCmd(client func() *ctlClient) *cobra.Command { + var ( + nodes []string + skipCP bool + releases string + perNode time.Duration + serverSvc string + ) + c := &cobra.Command{ + Use: "upgrade VERSION", + Short: "Upgrade this control plane, then every node, one at a time", + Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" + + "This machine first — run it with sudo on the control plane, since it\n" + + "replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" + + "turn: its agent fetches the release for its own platform, checks it\n" + + "against the release's checksums, swaps it in and restarts. Engines keep\n" + + "serving through all of it and are adopted again; no ssh into the nodes is\n" + + "needed.\n\n" + + "A node counts as done when it is back on the new version, ready, with as\n" + + "many ready replicas as before. One that is not within --timeout stops the\n" + + "upgrade there, so a bad release reaches one node rather than all of them.\n" + + "Nodes already on the version are skipped, so running it again resumes.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + v := args[0] + if !release.Tag.MatchString(v) { + return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v) + } + out := cmd.OutOrStdout() + ctx := cmd.Context() + if !skipCP { + if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil { + return err + } + } + return upgradeNodes(ctx, out, client, v, nodes, perNode) + }, + } + c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)") + c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes") + c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine") + c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back") + c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit") + return c +} + +func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error { + if version.Version == v { + fmt.Fprintf(out, "this machine already runs %s\n", v) + return nil + } + if os.Geteuid() != 0 { + return errors.New("replacing this machine's binary needs root: run it with sudo, " + + "or pass --skip-control-plane to upgrade only the nodes") + } + self, err := os.Executable() + if err != nil { + return err + } + fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v) + backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self) + if err != nil { + return err + } + fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup) + var before []adminapi.NodeView + _ = client().get(ctx, "/api/v1/nodes", &before) + if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command + return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b))) + } + fmt.Fprintf(out, " %s restarted; waiting for its nodes to reconnect\n", svc) + // Back when the admin API answers and every node that was ready is ready + // again: agents reconnect within seconds of the control plane returning. + want := 0 + for _, n := range before { + if n.State == "ready" { + want++ + } + } + deadline := time.Now().Add(2 * time.Minute) + for { + var now []adminapi.NodeView + if err := client().get(ctx, "/api/v1/nodes", &now); err == nil { + ready := 0 + for _, n := range now { + if n.State == "ready" { + ready++ + } + } + if ready >= want { + fmt.Fprintf(out, " back: %d of %d nodes ready\n", ready, len(now)) + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+ + "see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup) + } + time.Sleep(2 * time.Second) + } +} + +func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error { + var nodes []adminapi.NodeView + if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil { + return err + } + slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) }) + done, skipped := 0, 0 + for _, n := range nodes { + if len(only) > 0 && !slices.Contains(only, n.ID) { + continue + } + if n.AgentVersion == v { + skipped++ + continue + } + start := time.Now() + before := readyReplicasOn(ctx, client, n.ID) + fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v) + body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v}) + if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil { + return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID) + } + fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before) + deadline := time.Now().Add(perNode) + for { + var now []adminapi.NodeView + _ = client().get(ctx, "/api/v1/nodes", &now) + var cur adminapi.NodeView + for _, x := range now { + if x.ID == n.ID { + cur = x + } + } + after := readyReplicasOn(ctx, client, n.ID) + if cur.AgentVersion == v && cur.State == "ready" && after >= before { + fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n", + time.Since(start).Round(time.Second), after, before) + done++ + break + } + if time.Now().After(deadline) { + return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+ + "stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+ + "keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion, + cur.State, after, before, n.ID) + } + time.Sleep(2 * time.Second) + } + } + fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped) + return nil +} + +// readyReplicasOn counts the ready replicas with a shard on a node. +func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int { + var reps []adminapi.ReplicaView + if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil { + return 0 + } + n := 0 + for _, r := range reps { + if r.State != "ready" { + continue + } + for _, s := range r.Shards { + if s.Node == node { + n++ + break + } + } + } + return n +} diff --git a/internal/cli/ctl_upgrade_test.go b/internal/cli/ctl_upgrade_test.go new file mode 100644 index 0000000..1a8636e --- /dev/null +++ b/internal/cli/ctl_upgrade_test.go @@ -0,0 +1,88 @@ +package cli + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// fakeFleet is an admin API whose nodes come back on the version they are told +// to upgrade to — except those listed in stuck. +type fakeFleet struct { + mu sync.Mutex + versions map[string]string + stuck map[string]bool + told []string +} + +func (f *fakeFleet) server(t *testing.T) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + switch { + case r.URL.Path == "/api/v1/nodes": + var out []adminapi.NodeView + for id, v := range f.versions { + out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v}) + } + _ = json.NewEncoder(w).Encode(out) + case r.URL.Path == "/api/v1/replicas": + var out []adminapi.ReplicaView + for id := range f.versions { + out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready", + Shards: []adminapi.ShardView{{Node: id}}}) + } + _ = json.NewEncoder(w).Encode(out) + case strings.HasSuffix(r.URL.Path, "/upgrade"): + id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade") + var req adminapi.UpgradeRequest + _ = json.NewDecoder(r.Body).Decode(&req) + f.told = append(f.told, id) + if !f.stuck[id] { + f.versions[id] = req.Version + } + _, _ = w.Write([]byte(`{}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) { + f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}} + srv := f.server(t) + var out bytes.Buffer + err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second) + if err != nil { + t.Fatal(err) + } + if strings.Join(f.told, ",") != "a,c" { + t.Errorf("told %v; b was already on the version", f.told) + } + if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") { + t.Errorf("output:\n%s", out.String()) + } +} + +func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) { + // A bad release reaches one node, not all of them. + f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}} + srv := f.server(t) + var out bytes.Buffer + err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second) + if err == nil || !strings.Contains(err.Error(), "a did not come back") { + t.Fatalf("err %v", err) + } + if len(f.told) != 1 { + t.Errorf("went on to %v after a node failed to come back", f.told) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index cfe7d9a..f2501a8 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -173,6 +173,7 @@ func newServerCmd() *cobra.Command { } if admin != nil { api := adminapi.New(admin) + api.SetReleases(cfg.Releases) // Catalog changes need somewhere to persist, so this is wired // only when there is a store. Without one, the endpoints say so // rather than accepting a model that vanishes on restart. @@ -546,6 +547,19 @@ func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmg return ft.FabricTest(ctx, nodeID, req) } +// Upgrade forwards a node upgrade, as Logs does. +func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error { + d, err := r.get() + if err != nil { + return err + } + u, ok := d.(control.Upgrader) + if !ok { + return errors.New("this dispatcher cannot upgrade nodes") + } + return u.Upgrade(ctx, nodeID, version, releases) +} + func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error { d, err := r.get() if err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 01d6295..dc74f88 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -86,6 +86,11 @@ type Server struct { // Empty means /var/backups/llmgw-server. BackupDir string `koanf:"backup_dir"` + // Releases is where `llmgw ctl upgrade` fetches release binaries from, + // for this machine and — through their agents — every node. Empty means + // the project's own releases. + Releases string `koanf:"releases"` + Router Router `koanf:"router"` Control Control `koanf:"control"` Models []catalog.Model `koanf:"models"` diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 355f506..f5de421 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -10,6 +10,7 @@ import ( "context" "encoding/json" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" "io" "net/http" "slices" @@ -30,6 +31,9 @@ import ( // API serves the management endpoints. type API struct { + // releases is where release binaries are fetched from, for upgrades. + releases string + models Models capacity Capacity usage Usage @@ -74,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights) mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement) mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState) + mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode) mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode) mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest) mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs) @@ -1165,3 +1170,37 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) { PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros, }) } + +// SetReleases sets where upgrades fetch release binaries from. +func (a *API) SetReleases(base string) { a.releases = base } + +// UpgradeRequest names the release a node should run. +type UpgradeRequest struct { + Version string `json:"version"` +} + +func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) { + var req UpgradeRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error()) + return + } + if !release.Tag.MatchString(req.Version) { + writeError(w, http.StatusBadRequest, req.Version+" is not a release version") + return + } + u := a.ctl.Upgrader() + if u == nil { + writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes") + return + } + base := a.releases + if base == "" { + base = release.DefaultBase + } + if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil { + writeError(w, http.StatusBadGateway, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version}) +} diff --git a/internal/control/agentsvc/heartbeat_test.go b/internal/control/agentsvc/heartbeat_test.go index a5ab1b4..c612ed2 100644 --- a/internal/control/agentsvc/heartbeat_test.go +++ b/internal/control/agentsvc/heartbeat_test.go @@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) { t.Errorf("imex = %+v", n.IMEX) } } + +func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) { + s := newTestService() + c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})} + s.conns["n1"] = c + done := make(chan error, 1) + go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }() + + req := (<-c.send).GetUpgrade() + if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" { + t.Fatalf("sent %v", req) + } + s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{ + RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch", + }}}) + if err := <-done; err == nil || err.Error() != "checksum mismatch" { + t.Errorf("the agent's refusal was not passed on: %v", err) + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 91207c1..2655532 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -60,6 +60,8 @@ type Service struct { logMu sync.Mutex logWaits map[string]chan *llmgwv1.LogsReply logSeq uint64 + // upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu. + upgradeWaits map[string]chan *llmgwv1.UpgradeReply // scanWaits are the fabric scans awaiting a reply, by request id. scanWaits map[string]chan *llmgwv1.FabricScanReply testWaits map[string]chan *llmgwv1.FabricTestReply @@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) { ch <- m.FabricScan } + case *llmgwv1.AgentMessage_Upgrade: + s.logMu.Lock() + ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()] + delete(s.upgradeWaits, m.Upgrade.GetRequestId()) + s.logMu.Unlock() + if ok { + ch <- m.Upgrade + } + case *llmgwv1.AgentMessage_Logs: s.logMu.Lock() ch, ok := s.logWaits[m.Logs.GetRequestId()] @@ -842,3 +853,44 @@ func installable(reg *llmgwv1.Register) []string { // The admin API finds the bandwidth test by asserting this; a signature that // drifted would make it quietly nil. var _ fabrictest.Asker = (*Service)(nil) + +// Upgrade tells a node's agent to replace its binary with a release and +// restart, and returns once the agent says the new binary is in place — or +// why it is not. The agent then exits and comes back on the new version; the +// caller watches for that in the inventory. +func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error { + s.logMu.Lock() + if s.upgradeWaits == nil { + s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{} + } + s.logSeq++ + id := fmt.Sprintf("upgrade-%d", s.logSeq) + ch := make(chan *llmgwv1.UpgradeReply, 1) + s.upgradeWaits[id] = ch + s.logMu.Unlock() + defer func() { + s.logMu.Lock() + delete(s.upgradeWaits, id) + s.logMu.Unlock() + }() + + if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{ + Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases}, + }}); err != nil { + return err + } + s.log.Info("told a node to upgrade", "node", nodeID, "version", version) + // A download of some tens of megabytes on a slow link, and no longer: an + // agent too old to know this message never answers. + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + select { + case r := <-ch: + if r.GetError() != "" { + return errors.New(r.GetError()) + } + return nil + case <-ctx.Done(): + return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID) + } +} diff --git a/internal/control/control.go b/internal/control/control.go index 38727b2..d742eda 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -211,6 +211,17 @@ func (c *Controller) WeightRemover() WeightRemover { return r } +// Upgrader can tell a node's agent to upgrade itself. +type Upgrader interface { + Upgrade(ctx context.Context, nodeID, version, releases string) error +} + +// Upgrader returns the dispatcher's upgrade capability, or nil. +func (c *Controller) Upgrader() Upgrader { + u, _ := c.disp.(Upgrader) + return u +} + // LogReader can ask a node for the recent lines of one of its logs. type LogReader interface { Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error) diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 6500814..af615de 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -18,6 +18,7 @@ import ( "encoding/pem" "errors" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/release" "net" "os" "path/filepath" @@ -335,7 +336,7 @@ func certNames(path string) ([]string, error) { // releaseTag is a version CI published: v0.10.4, or a release candidate of // it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no // release to download. -var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`) +var releaseTag = release.Tag func (r *Runner) binary(ctx context.Context, res *Result) error { v := r.Opts.Version diff --git a/internal/release/release.go b/internal/release/release.go new file mode 100644 index 0000000..d1c64f9 --- /dev/null +++ b/internal/release/release.go @@ -0,0 +1,148 @@ +// Package release fetches and installs llmgw's own release binaries. +// +// One implementation for everywhere a binary is replaced — the control plane +// upgrading itself, and an agent told to upgrade by it — so the checks are +// the same on both: the release's own checksum is verified before anything is +// installed, the old binary is kept beside the new one, and the swap is a +// rename, so a machine is never left without a binary that runs. +package release + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// DefaultBase is where CI publishes releases. +const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download" + +// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2. +var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`) + +// AssetName is the release file for a platform. +func AssetName(version, goos, goarch string) string { + return "llmgw-" + version + "-" + goos + "-" + goarch +} + +// Install fetches version for this platform from base, checks it against the +// release's checksums, and puts it at target, keeping the old one as +// target.bak-pre-. It returns the backup's path. +func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) { + if !Tag.MatchString(version) { + return "", fmt.Errorf("%q is not a release version", version) + } + if client == nil { + client = &http.Client{Timeout: 10 * time.Minute} + } + target, err := filepath.EvalSymlinks(target) + if err != nil { + return "", err + } + dir := filepath.Dir(target) + asset := AssetName(version, goos, goarch) + root := strings.TrimSuffix(base, "/") + "/" + version + "/" + + want, err := checksum(ctx, client, root+"checksums.txt", asset) + if err != nil { + return "", err + } + // Beside the target, so the final rename is on one filesystem and atomic. + tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + got, err := download(ctx, client, root+asset, tmp) + if cerr := tmp.Close(); err == nil { + err = cerr + } + if err != nil { + return "", err + } + if got != want { + return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)", + asset, got[:12], want[:12]) + } + if err := os.Chmod(tmp.Name(), 0o755); err != nil { + return "", err + } + backup := target + ".bak-pre-" + version + if err := copyFile(target, backup); err != nil { + return "", fmt.Errorf("keeping the old binary: %w", err) + } + if err := os.Rename(tmp.Name(), target); err != nil { + return "", err + } + return backup, nil +} + +func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) { + body, err := get(ctx, client, url) + if err != nil { + return "", err + } + defer body.Close() + sc := bufio.NewScanner(io.LimitReader(body, 1<<20)) + for sc.Scan() { + f := strings.Fields(sc.Text()) + if len(f) == 2 && f[1] == asset && len(f[0]) == 64 { + return f[0], nil + } + } + return "", fmt.Errorf("the release's checksums list no %s", asset) +} + +func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) { + body, err := get(ctx, client, url) + if err != nil { + return "", err + } + defer body.Close() + h := sha256.New() + if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil { + return "", fmt.Errorf("downloading %s: %w", url, err) + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching %s: %w", url, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("fetching %s: %s", url, resp.Status) + } + return resp.Body, nil +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) //nolint:gosec // our own binary + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} diff --git a/internal/release/release_test.go b/internal/release/release_test.go new file mode 100644 index 0000000..b57f0bd --- /dev/null +++ b/internal/release/release_test.go @@ -0,0 +1,61 @@ +package release + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func releaseServer(t *testing.T, version, asset string, body []byte, sum string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/" + version + "/checksums.txt": + _, _ = w.Write([]byte(sum + " " + asset + "\n")) + case "/" + version + "/" + asset: + _, _ = w.Write(body) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "llmgw") + _ = os.WriteFile(target, []byte("old"), 0o755) + body := []byte("new binary") + sum := sha256.Sum256(body) + asset := AssetName("v1.2.3", "linux", "amd64") + + srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:])) + backup, err := Install(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "amd64", target) + if err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(target); string(b) != "new binary" { + t.Errorf("target holds %q", b) + } + if b, _ := os.ReadFile(backup); string(b) != "old" { + t.Errorf("the old binary was not kept: %q", b) + } + + // A download that does not match its checksum changes nothing. + bad := releaseServer(t, "v1.2.4", AssetName("v1.2.4", "linux", "amd64"), []byte("tampered"), + hex.EncodeToString(sum[:])) + if _, err := Install(t.Context(), bad.Client(), bad.URL, "v1.2.4", "linux", "amd64", target); err == nil { + t.Fatal("a binary not matching its checksum was installed") + } + if b, _ := os.ReadFile(target); string(b) != "new binary" { + t.Errorf("a failed install touched the binary: %q", b) + } + if _, err := Install(t.Context(), nil, srv.URL, "../../x", "linux", "amd64", target); err == nil { + t.Error("a version that is not one was fetched") + } +}