Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
+241
-57
@@ -104,6 +104,7 @@ type AgentMessage struct {
|
||||
// *AgentMessage_Logs
|
||||
// *AgentMessage_FabricScan
|
||||
// *AgentMessage_FabricTest
|
||||
// *AgentMessage_Upgrade
|
||||
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -200,6 +201,15 @@ func (x *AgentMessage) GetFabricTest() *FabricTestReply {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AgentMessage) GetUpgrade() *UpgradeReply {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*AgentMessage_Upgrade); ok {
|
||||
return x.Upgrade
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isAgentMessage_Msg interface {
|
||||
isAgentMessage_Msg()
|
||||
}
|
||||
@@ -228,6 +238,10 @@ type AgentMessage_FabricTest struct {
|
||||
FabricTest *FabricTestReply `protobuf:"bytes,6,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
|
||||
}
|
||||
|
||||
type AgentMessage_Upgrade struct {
|
||||
Upgrade *UpgradeReply `protobuf:"bytes,7,opt,name=upgrade,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*AgentMessage_Register) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
|
||||
@@ -240,6 +254,8 @@ func (*AgentMessage_FabricScan) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_FabricTest) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_Upgrade) isAgentMessage_Msg() {}
|
||||
|
||||
// ControlMessage is anything the control plane tells a node.
|
||||
type ControlMessage struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -254,6 +270,7 @@ type ControlMessage struct {
|
||||
// *ControlMessage_Logs
|
||||
// *ControlMessage_FabricScan
|
||||
// *ControlMessage_FabricTest
|
||||
// *ControlMessage_Upgrade
|
||||
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -377,6 +394,15 @@ func (x *ControlMessage) GetFabricTest() *FabricTestRequest {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetUpgrade() *UpgradeAgent {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*ControlMessage_Upgrade); ok {
|
||||
return x.Upgrade
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isControlMessage_Msg interface {
|
||||
isControlMessage_Msg()
|
||||
}
|
||||
@@ -417,6 +443,10 @@ type ControlMessage_FabricTest struct {
|
||||
FabricTest *FabricTestRequest `protobuf:"bytes,9,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
|
||||
}
|
||||
|
||||
type ControlMessage_Upgrade struct {
|
||||
Upgrade *UpgradeAgent `protobuf:"bytes,10,opt,name=upgrade,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*ControlMessage_Start) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_Stop) isControlMessage_Msg() {}
|
||||
@@ -435,6 +465,8 @@ func (*ControlMessage_FabricScan) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_FabricTest) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_Upgrade) isControlMessage_Msg() {}
|
||||
|
||||
// Register is the first message on every stream. It carries the node's full
|
||||
// hardware inventory, so the control plane needs no separate discovery call and
|
||||
// a reconnecting agent automatically refreshes what the scheduler knows.
|
||||
@@ -3460,11 +3492,144 @@ func (x *FabricTestReply) GetTookMs() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// UpgradeAgent tells an agent to replace its own binary with a release and
|
||||
// restart.
|
||||
//
|
||||
// The agent fetches the release for its own platform and checks it against
|
||||
// the release's checksums before replacing anything, keeps the old binary
|
||||
// beside the new, then exits so systemd starts the new one. Engines keep
|
||||
// running throughout, as for any agent restart, and are adopted on the way
|
||||
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
||||
// no ssh into it.
|
||||
type UpgradeAgent struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
||||
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
|
||||
// releases is the base URL release files are downloaded from.
|
||||
Releases string `protobuf:"bytes,3,opt,name=releases,proto3" json:"releases,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) Reset() {
|
||||
*x = UpgradeAgent{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UpgradeAgent) ProtoMessage() {}
|
||||
|
||||
func (x *UpgradeAgent) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead.
|
||||
func (*UpgradeAgent) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33}
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) GetVersion() string {
|
||||
if x != nil {
|
||||
return x.Version
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) GetReleases() string {
|
||||
if x != nil {
|
||||
return x.Releases
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// UpgradeReply says whether the new binary is in place. Sent before the
|
||||
// agent exits; an empty error means it will be back on the new version.
|
||||
type UpgradeReply struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
|
||||
Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) Reset() {
|
||||
*x = UpgradeReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UpgradeReply) ProtoMessage() {}
|
||||
|
||||
func (x *UpgradeReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead.
|
||||
func (*UpgradeReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34}
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) GetVersion() string {
|
||||
if x != nil {
|
||||
return x.Version
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) GetError() string {
|
||||
if x != nil {
|
||||
return x.Error
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd2\x02\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x86\x03\n" +
|
||||
"\fAgentMessage\x120\n" +
|
||||
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
|
||||
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
|
||||
@@ -3473,8 +3638,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" +
|
||||
"fabricScan\x12<\n" +
|
||||
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
|
||||
"fabricTestB\x05\n" +
|
||||
"\x03msg\"\xe5\x03\n" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgradeB\x05\n" +
|
||||
"\x03msg\"\x99\x04\n" +
|
||||
"\x0eControlMessage\x12,\n" +
|
||||
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
@@ -3486,7 +3652,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
|
||||
"fabricScan\x12>\n" +
|
||||
"\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" +
|
||||
"fabricTestB\x05\n" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\n" +
|
||||
" \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" +
|
||||
"\x03msg\"\xe3\x06\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
@@ -3795,7 +3963,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04gbps\x18\x02 \x01(\x01R\x04gbps\x12\x16\n" +
|
||||
"\x06output\x18\x03 \x01(\tR\x06output\x12\x14\n" +
|
||||
"\x05error\x18\x04 \x01(\tR\x05error\x12\x17\n" +
|
||||
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs2O\n" +
|
||||
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs\"c\n" +
|
||||
"\fUpgradeAgent\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x1a\n" +
|
||||
"\breleases\x18\x03 \x01(\tR\breleases\"]\n" +
|
||||
"\fUpgradeReply\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -3814,7 +3992,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 41)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -3850,12 +4028,14 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
|
||||
(*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest
|
||||
(*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply
|
||||
nil, // 34: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 35: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 36: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 37: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 38: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 39: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*UpgradeAgent)(nil), // 34: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 35: llmgw.v1.UpgradeReply
|
||||
nil, // 36: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 37: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 38: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 39: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 40: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 41: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -3864,50 +4044,52 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
|
||||
33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
|
||||
22, // 6: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 7: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 8: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 9: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 10: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 11: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 12: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 13: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 14: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
34, // 15: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 16: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 17: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
35, // 18: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 19: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 20: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 21: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 22: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
14, // 23: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 24: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
36, // 25: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 26: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
37, // 27: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 28: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 29: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 30: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
38, // 31: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 32: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 33: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 34: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 35: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 36: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 37: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 38: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 39: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 40: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 41: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
39, // 42: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 43: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 44: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
44, // [44:45] is the sub-list for method output_type
|
||||
43, // [43:44] is the sub-list for method input_type
|
||||
43, // [43:43] is the sub-list for extension type_name
|
||||
43, // [43:43] is the sub-list for extension extendee
|
||||
0, // [0:43] is the sub-list for field type_name
|
||||
35, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply
|
||||
22, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
34, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
36, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
37, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
14, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
38, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
39, // 29: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 30: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 31: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 32: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
40, // 33: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 34: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 35: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 36: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 37: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 38: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 39: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 40: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 41: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 42: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 43: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
41, // 44: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 45: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 46: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
46, // [46:47] is the sub-list for method output_type
|
||||
45, // [45:46] is the sub-list for method input_type
|
||||
45, // [45:45] is the sub-list for extension type_name
|
||||
45, // [45:45] is the sub-list for extension extendee
|
||||
0, // [0:45] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -3922,6 +4104,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*AgentMessage_Logs)(nil),
|
||||
(*AgentMessage_FabricScan)(nil),
|
||||
(*AgentMessage_FabricTest)(nil),
|
||||
(*AgentMessage_Upgrade)(nil),
|
||||
}
|
||||
file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{
|
||||
(*ControlMessage_Start)(nil),
|
||||
@@ -3933,6 +4116,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*ControlMessage_Logs)(nil),
|
||||
(*ControlMessage_FabricScan)(nil),
|
||||
(*ControlMessage_FabricTest)(nil),
|
||||
(*ControlMessage_Upgrade)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@@ -3940,7 +4124,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 39,
|
||||
NumMessages: 41,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -26,6 +26,7 @@ message AgentMessage {
|
||||
LogsReply logs = 4;
|
||||
FabricScanReply fabric_scan = 5;
|
||||
FabricTestReply fabric_test = 6;
|
||||
UpgradeReply upgrade = 7;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +42,7 @@ message ControlMessage {
|
||||
LogsRequest logs = 7;
|
||||
FabricScanRequest fabric_scan = 8;
|
||||
FabricTestRequest fabric_test = 9;
|
||||
UpgradeAgent upgrade = 10;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -649,3 +651,28 @@ message FabricTestReply {
|
||||
string error = 4;
|
||||
int64 took_ms = 5;
|
||||
}
|
||||
|
||||
// UpgradeAgent tells an agent to replace its own binary with a release and
|
||||
// restart.
|
||||
//
|
||||
// The agent fetches the release for its own platform and checks it against
|
||||
// the release's checksums before replacing anything, keeps the old binary
|
||||
// beside the new, then exits so systemd starts the new one. Engines keep
|
||||
// running throughout, as for any agent restart, and are adopted on the way
|
||||
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
||||
// no ssh into it.
|
||||
message UpgradeAgent {
|
||||
string request_id = 1;
|
||||
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
||||
string version = 2;
|
||||
// releases is the base URL release files are downloaded from.
|
||||
string releases = 3;
|
||||
}
|
||||
|
||||
// UpgradeReply says whether the new binary is in place. Sent before the
|
||||
// agent exits; an empty error means it will be back on the new version.
|
||||
message UpgradeReply {
|
||||
string request_id = 1;
|
||||
string version = 2;
|
||||
string error = 3;
|
||||
}
|
||||
Reference in new issue
Block a user