Upgrade the cluster with one command, over the agents' own streams
Release / build (push) Successful in 1m5s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m42s

Every upgrade so far has been a script of mine copying a binary to each
machine over ssh, restarting it and polling until it came back — which is
the job, but not something the product could do, and which needed ssh into
every node.

`sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine
first: the release for its platform, checked against the release's
checksums, the old binary kept beside it, llmgw-server restarted, and a wait
until every node that was ready has reconnected. Then each node in turn: the
control plane tells its agent the version over the stream it already holds;
the agent fetches and checks the release itself, swaps it in with a rename,
answers, and exits with status 75 so systemd starts the new binary. Engines
keep running, as for any agent restart, and are adopted again.

A node is done when it is back on the new version, ready, with as many ready
replicas as before; one that is not within the timeout stops the run before
the next node, so a bad release reaches one machine. Nodes already on the
version are skipped, so running it again resumes.

The fetch-and-verify is one package now, shared with onboarding's idea of
what a release version looks like.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 02:54:36 +02:00
1 parent 3bf4b7b709
commit 116e75f8cb
19 files changed
+1082 -90

No files matched your search

+11
View File
@@ -77,6 +77,17 @@ more than 26 hours old or there has never been one. docs/install.md now says how
to restore, including why running engines survive it: the engine-key secret
comes back with the store.
### Upgrading from one command
`sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that
machine and then every node in turn. Each agent is told the version over its
existing stream, fetches the release for its own platform, checks it against
the release's checksums, keeps the old binary beside the new and restarts,
while its engines keep serving. A node that does not come back on the new
version, ready, with its replicas, stops the upgrade before the next one.
Running it again resumes. Agents from before this have to be moved by hand
once.
### Following a request
Every response carries `X-Request-Id` — the client's own when it sends a plain
+241 -57
View File
@@ -104,6 +104,7 @@ type AgentMessage struct {
// *AgentMessage_Logs
// *AgentMessage_FabricScan
// *AgentMessage_FabricTest
// *AgentMessage_Upgrade
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -200,6 +201,15 @@ func (x *AgentMessage) GetFabricTest() *FabricTestReply {
return nil
}
func (x *AgentMessage) GetUpgrade() *UpgradeReply {
if x != nil {
if x, ok := x.Msg.(*AgentMessage_Upgrade); ok {
return x.Upgrade
}
}
return nil
}
type isAgentMessage_Msg interface {
isAgentMessage_Msg()
}
@@ -228,6 +238,10 @@ type AgentMessage_FabricTest struct {
FabricTest *FabricTestReply `protobuf:"bytes,6,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
}
type AgentMessage_Upgrade struct {
Upgrade *UpgradeReply `protobuf:"bytes,7,opt,name=upgrade,proto3,oneof"`
}
func (*AgentMessage_Register) isAgentMessage_Msg() {}
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
@@ -240,6 +254,8 @@ func (*AgentMessage_FabricScan) isAgentMessage_Msg() {}
func (*AgentMessage_FabricTest) isAgentMessage_Msg() {}
func (*AgentMessage_Upgrade) isAgentMessage_Msg() {}
// ControlMessage is anything the control plane tells a node.
type ControlMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -254,6 +270,7 @@ type ControlMessage struct {
// *ControlMessage_Logs
// *ControlMessage_FabricScan
// *ControlMessage_FabricTest
// *ControlMessage_Upgrade
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -377,6 +394,15 @@ func (x *ControlMessage) GetFabricTest() *FabricTestRequest {
return nil
}
func (x *ControlMessage) GetUpgrade() *UpgradeAgent {
if x != nil {
if x, ok := x.Msg.(*ControlMessage_Upgrade); ok {
return x.Upgrade
}
}
return nil
}
type isControlMessage_Msg interface {
isControlMessage_Msg()
}
@@ -417,6 +443,10 @@ type ControlMessage_FabricTest struct {
FabricTest *FabricTestRequest `protobuf:"bytes,9,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
}
type ControlMessage_Upgrade struct {
Upgrade *UpgradeAgent `protobuf:"bytes,10,opt,name=upgrade,proto3,oneof"`
}
func (*ControlMessage_Start) isControlMessage_Msg() {}
func (*ControlMessage_Stop) isControlMessage_Msg() {}
@@ -435,6 +465,8 @@ func (*ControlMessage_FabricScan) isControlMessage_Msg() {}
func (*ControlMessage_FabricTest) isControlMessage_Msg() {}
func (*ControlMessage_Upgrade) isControlMessage_Msg() {}
// Register is the first message on every stream. It carries the node's full
// hardware inventory, so the control plane needs no separate discovery call and
// a reconnecting agent automatically refreshes what the scheduler knows.
@@ -3460,11 +3492,144 @@ func (x *FabricTestReply) GetTookMs() int64 {
return 0
}
// UpgradeAgent tells an agent to replace its own binary with a release and
// restart.
//
// The agent fetches the release for its own platform and checks it against
// the release's checksums before replacing anything, keeps the old binary
// beside the new, then exits so systemd starts the new one. Engines keep
// running throughout, as for any agent restart, and are adopted on the way
// back. Over the stream the agent already holds, so upgrading a fleet needs
// no ssh into it.
type UpgradeAgent struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
// version must be a release: v0.10.4 or v0.10.4-rc.2.
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
// releases is the base URL release files are downloaded from.
Releases string `protobuf:"bytes,3,opt,name=releases,proto3" json:"releases,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *UpgradeAgent) Reset() {
*x = UpgradeAgent{}
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *UpgradeAgent) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UpgradeAgent) ProtoMessage() {}
func (x *UpgradeAgent) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead.
func (*UpgradeAgent) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33}
}
func (x *UpgradeAgent) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
func (x *UpgradeAgent) GetVersion() string {
if x != nil {
return x.Version
}
return ""
}
func (x *UpgradeAgent) GetReleases() string {
if x != nil {
return x.Releases
}
return ""
}
// UpgradeReply says whether the new binary is in place. Sent before the
// agent exits; an empty error means it will be back on the new version.
type UpgradeReply struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *UpgradeReply) Reset() {
*x = UpgradeReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *UpgradeReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UpgradeReply) ProtoMessage() {}
func (x *UpgradeReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead.
func (*UpgradeReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34}
}
func (x *UpgradeReply) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
func (x *UpgradeReply) GetVersion() string {
if x != nil {
return x.Version
}
return ""
}
func (x *UpgradeReply) GetError() string {
if x != nil {
return x.Error
}
return ""
}
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd2\x02\n" +
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x86\x03\n" +
"\fAgentMessage\x120\n" +
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
@@ -3473,8 +3638,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" +
"fabricScan\x12<\n" +
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
"fabricTestB\x05\n" +
"\x03msg\"\xe5\x03\n" +
"fabricTest\x122\n" +
"\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgradeB\x05\n" +
"\x03msg\"\x99\x04\n" +
"\x0eControlMessage\x12,\n" +
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
@@ -3486,7 +3652,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
"fabricScan\x12>\n" +
"\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" +
"fabricTestB\x05\n" +
"fabricTest\x122\n" +
"\aupgrade\x18\n" +
" \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" +
"\x03msg\"\xe3\x06\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
@@ -3795,7 +3963,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04gbps\x18\x02 \x01(\x01R\x04gbps\x12\x16\n" +
"\x06output\x18\x03 \x01(\tR\x06output\x12\x14\n" +
"\x05error\x18\x04 \x01(\tR\x05error\x12\x17\n" +
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs2O\n" +
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs\"c\n" +
"\fUpgradeAgent\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
"\aversion\x18\x02 \x01(\tR\aversion\x12\x1a\n" +
"\breleases\x18\x03 \x01(\tR\breleases\"]\n" +
"\fUpgradeReply\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
"\fAgentService\x12?\n" +
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
"\fcom.llmgw.v1B\n" +
@@ -3814,7 +3992,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 41)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -3850,12 +4028,14 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
(*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest
(*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply
nil, // 34: llmgw.v1.Register.LabelsEntry
nil, // 35: llmgw.v1.Register.EngineVersionsEntry
nil, // 36: llmgw.v1.GPU.NicPathsEntry
nil, // 37: llmgw.v1.IBPort.CountersEntry
nil, // 38: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 39: llmgw.v1.EngineProfile.EnvEntry
(*UpgradeAgent)(nil), // 34: llmgw.v1.UpgradeAgent
(*UpgradeReply)(nil), // 35: llmgw.v1.UpgradeReply
nil, // 36: llmgw.v1.Register.LabelsEntry
nil, // 37: llmgw.v1.Register.EngineVersionsEntry
nil, // 38: llmgw.v1.GPU.NicPathsEntry
nil, // 39: llmgw.v1.IBPort.CountersEntry
nil, // 40: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 41: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
@@ -3864,50 +4044,52 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
22, // 6: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
25, // 7: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
26, // 8: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
27, // 9: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
24, // 10: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
17, // 11: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
28, // 12: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
30, // 13: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
32, // 14: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
34, // 15: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 16: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 17: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
35, // 18: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 19: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 20: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 21: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 22: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
14, // 23: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
0, // 24: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
36, // 25: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 26: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
37, // 27: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 28: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 29: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 30: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
38, // 31: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
19, // 32: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 33: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 34: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 35: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 36: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
14, // 37: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
18, // 38: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 39: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
21, // 40: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
23, // 41: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
39, // 42: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 43: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 44: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
44, // [44:45] is the sub-list for method output_type
43, // [43:44] is the sub-list for method input_type
43, // [43:43] is the sub-list for extension type_name
43, // [43:43] is the sub-list for extension extendee
0, // [0:43] is the sub-list for field type_name
35, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply
22, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
25, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
26, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
27, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
24, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
17, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
28, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
30, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
32, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
34, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
36, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
37, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
14, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
38, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
39, // 29: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 30: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 31: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 32: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
40, // 33: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
19, // 34: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 35: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 36: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 37: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 38: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
14, // 39: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
18, // 40: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 41: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
21, // 42: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
23, // 43: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
41, // 44: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 45: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 46: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
46, // [46:47] is the sub-list for method output_type
45, // [45:46] is the sub-list for method input_type
45, // [45:45] is the sub-list for extension type_name
45, // [45:45] is the sub-list for extension extendee
0, // [0:45] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -3922,6 +4104,7 @@ func file_llmgw_v1_agent_proto_init() {
(*AgentMessage_Logs)(nil),
(*AgentMessage_FabricScan)(nil),
(*AgentMessage_FabricTest)(nil),
(*AgentMessage_Upgrade)(nil),
}
file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{
(*ControlMessage_Start)(nil),
@@ -3933,6 +4116,7 @@ func file_llmgw_v1_agent_proto_init() {
(*ControlMessage_Logs)(nil),
(*ControlMessage_FabricScan)(nil),
(*ControlMessage_FabricTest)(nil),
(*ControlMessage_Upgrade)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -3940,7 +4124,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 39,
NumMessages: 41,
NumExtensions: 0,
NumServices: 1,
},
+27
View File
@@ -26,6 +26,7 @@ message AgentMessage {
LogsReply logs = 4;
FabricScanReply fabric_scan = 5;
FabricTestReply fabric_test = 6;
UpgradeReply upgrade = 7;
}
}
@@ -41,6 +42,7 @@ message ControlMessage {
LogsRequest logs = 7;
FabricScanRequest fabric_scan = 8;
FabricTestRequest fabric_test = 9;
UpgradeAgent upgrade = 10;
}
}
@@ -649,3 +651,28 @@ message FabricTestReply {
string error = 4;
int64 took_ms = 5;
}
// UpgradeAgent tells an agent to replace its own binary with a release and
// restart.
//
// The agent fetches the release for its own platform and checks it against
// the release's checksums before replacing anything, keeps the old binary
// beside the new, then exits so systemd starts the new one. Engines keep
// running throughout, as for any agent restart, and are adopted on the way
// back. Over the stream the agent already holds, so upgrading a fleet needs
// no ssh into it.
message UpgradeAgent {
string request_id = 1;
// version must be a release: v0.10.4 or v0.10.4-rc.2.
string version = 2;
// releases is the base URL release files are downloaded from.
string releases = 3;
}
// UpgradeReply says whether the new binary is in place. Sent before the
// agent exits; an empty error means it will be back on the new version.
message UpgradeReply {
string request_id = 1;
string version = 2;
string error = 3;
}
+29 -32
View File
@@ -909,55 +909,52 @@ refuses every request.
## Upgrading
One binary on every machine, so an upgrade is a copy and a restart. The part
worth care is the copy, not the order.
On the control plane:
```
sudo llmgw ctl upgrade v0.10.4
```
It upgrades this machine first — fetching the release for its platform,
checking it against the release's checksums, keeping the old binary as
`/usr/local/bin/llmgw.bak-pre-<version>`, restarting `llmgw-server` and waiting
for every node to reconnect — and then each node in turn. A node's agent is told
the version over the stream it already holds, fetches and checks the release
itself, swaps it in and restarts; no ssh into the nodes is needed. Release
candidates (`v0.10.4-rc.2`) work the same way.
A node counts as done when it is back on the new version, ready, with as many
ready replicas as before. One that is not within `--timeout` (three minutes)
stops the upgrade there, so a bad release reaches one node rather than all of
them; `llmgw ctl logs <node>` says why, and the node keeps its previous binary
beside the new one. Nodes already on the version are skipped, so running the
command again resumes where it stopped. `--nodes a,b` limits it to some nodes,
and `--skip-control-plane` leaves this machine alone.
**Nothing stops serving.** Engines are started with `KillMode=process`, so they
keep running when the agent restarts, and the agent reports them back on its
next connection. The control plane holds no replica state across its own
restart either — it rebuilds the fleet from what the agents report. A rolling
upgrade of the whole cluster interrupts no request that is not in flight at the
moment of the restart.
restart either — it rebuilds the fleet from what the agents report.
**A mixed fleet is fine while you roll.** The agent protocol only ever gains
fields, and both sides treat an absent one as unknown rather than as a default.
Order does not matter; finish the roll the same day anyway, because
`ctl nodes` reports each agent's version and a fleet that disagrees with itself
is one more thing to hold in your head.
```
VERSION=v0.9.0
# copy to every machine first, install nowhere yet
for h in 10.0.0.140 10.0.0.141 10.0.0.142 10.0.0.146; do
ssh $h 'cat > /tmp/llmgw-new; stat -c %s /tmp/llmgw-new' < llmgw-$VERSION-linux-amd64
done
```
**Check every size before installing anything.** This is not defensive writing:
on a host whose login shell prints anything, `scp` fails outright, and the
obvious workaround — `ssh -n host 'cat > file'` — points stdin at `/dev/null`
and writes a *zero-byte file* while reporting success. Installing that on a
fleet takes the fleet down. Compare the number on each machine with the local
one, then:
```
sudo install -m0755 /tmp/llmgw-new /usr/local/bin/llmgw
sudo systemctl restart llmgw-agent # or llmgw-server, or llmgw-mirror
llmgw --version
```
Agents older than v0.10.4 cannot upgrade themselves: move those by hand once,
installing the release binary and restarting `llmgw-agent`, and every upgrade
after that is the command above. The mirror, which is not an agent, is
upgraded the same way by hand.
Then confirm what you have, rather than assuming:
```
llmgw ctl nodes # every agent back, LAST SEEN in seconds
llmgw ctl nodes # every agent on the new version, LAST SEEN in seconds
llmgw ctl replicas # same replicas, adopted rather than replaced
```
A replica may come back marked **`ready (stale profile)`**. That means it is
running a profile that has since been edited — the reconciler leaves a serving
replica alone on purpose, because churning one costs more than the difference.
Restart it when you want the new profile to take effect; the flag is there so
the choice is yours rather than silent.
`llmgw ctl restart <replica>` replaces it with one on the new profile, starting
the new one before stopping the old.
## Backing up
+9
View File
@@ -170,6 +170,7 @@ rather than repeated.
| `mirror` | `Mirror` | [Server](#server) |
| `engines` | `map[string]string` | [Server](#server) |
| `backup_dir` | `string` | [Server](#server) |
| `releases` | `string` | [Server](#server) |
| `router` | `Router` | [Server](#server) |
| `control` | `Control` | [Server](#server) |
| `models` | `[]catalog.Model` | [Server](#server) |
@@ -1274,6 +1275,14 @@ BackupDir is where `llmgw server backup` writes, and where the newest
archive's age is read for llmgw_backup_last_success_timestamp_seconds.
Empty means /var/backups/llmgw-server.
### `releases`
`string`
Releases is where `llmgw ctl upgrade` fetches release binaries from,
for this machine and — through their agents — every node. Empty means
the project's own releases.
### `router`
`Router`
+50
View File
@@ -8,6 +8,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"log/slog"
"maps"
@@ -18,6 +19,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
@@ -1119,6 +1121,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
a.removeWeights(m.RemoveWeights.GetPaths())
case *llmgwv1.ControlMessage_Provision:
a.setProvision(m.Provision)
case *llmgwv1.ControlMessage_Upgrade:
go a.upgrade(ctx, m.Upgrade, out)
case *llmgwv1.ControlMessage_Logs:
reply := a.logsReply(m.Logs)
// Off the receive loop: a full send queue must not stop commands being
@@ -1874,3 +1878,49 @@ func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
m := discoverIMEX(ctx)
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
}
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
// Restart=on-failure starts the new binary, and distinct, so the journal says
// why the agent went.
const upgradeExit = 75
// exitForUpgrade ends the process; a variable so a test can see it called
// rather than have its own process end.
var exitForUpgrade = func() { os.Exit(upgradeExit) }
// upgradeTarget is the binary an upgrade replaces; a variable so a test
// replaces a file of its own rather than the test binary running it.
var upgradeTarget = os.Executable
// upgrade replaces this agent's binary with a release and restarts into it.
//
// Engines are not touched: an agent restart leaves them running and adopts
// them again, which is what makes upgrading a node something that can be done
// while it serves.
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
send := func() {
select {
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
case <-ctx.Done():
}
}
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
self, err := upgradeTarget()
if err == nil {
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
cancel()
}
if err != nil {
reply.Error = err.Error()
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
send()
return
}
send()
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
// A moment for the reply to leave before the stream goes.
time.Sleep(500 * time.Millisecond)
exitForUpgrade()
}
+78
View File
@@ -0,0 +1,78 @@
package agent
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
body := []byte("the new agent")
sum := sha256.Sum256(body)
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v9.9.9/checksums.txt":
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
case "/v9.9.9/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
bin := filepath.Join(t.TempDir(), "llmgw")
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
exited := make(chan struct{}, 1)
oldTarget, oldExit := upgradeTarget, exitForUpgrade
upgradeTarget = func() (string, error) { return bin, nil }
exitForUpgrade = func() { exited <- struct{}{} }
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
cfg.Superv.WorkDir = t.TempDir()
a, err := New(cfg, nil, nil)
if err != nil {
t.Fatal(err)
}
out := make(chan *llmgwv1.AgentMessage, 1)
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
select {
case m := <-out:
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
t.Fatalf("reply %+v", r)
}
case <-time.After(10 * time.Second):
t.Fatal("no reply")
}
select {
case <-exited:
case <-time.After(5 * time.Second):
t.Fatal("the agent did not restart into the new binary")
}
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
t.Errorf("binary holds %q", b)
}
// A version with no release answers with why, and does not restart.
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
if r := (<-out).GetUpgrade(); r.GetError() == "" {
t.Error("an upgrade to a release that does not exist reported success")
}
select {
case <-exited:
t.Error("a failed upgrade restarted the agent")
case <-time.After(time.Second):
}
}
+1
View File
@@ -141,6 +141,7 @@ func newCtlCmd() *cobra.Command {
newScaleCmd(client),
newRestartCmd(client),
newRetryCmd(client),
newUpgradeCmd(client),
newEngineCheckCmd(),
newToolCheckCmd(),
newWeightsCmd(client, &asJSON),
+197
View File
@@ -0,0 +1,197 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"runtime"
"slices"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
var (
nodes []string
skipCP bool
releases string
perNode time.Duration
serverSvc string
)
c := &cobra.Command{
Use: "upgrade VERSION",
Short: "Upgrade this control plane, then every node, one at a time",
Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
"This machine first — run it with sudo on the control plane, since it\n" +
"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
"turn: its agent fetches the release for its own platform, checks it\n" +
"against the release's checksums, swaps it in and restarts. Engines keep\n" +
"serving through all of it and are adopted again; no ssh into the nodes is\n" +
"needed.\n\n" +
"A node counts as done when it is back on the new version, ready, with as\n" +
"many ready replicas as before. One that is not within --timeout stops the\n" +
"upgrade there, so a bad release reaches one node rather than all of them.\n" +
"Nodes already on the version are skipped, so running it again resumes.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
v := args[0]
if !release.Tag.MatchString(v) {
return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
}
out := cmd.OutOrStdout()
ctx := cmd.Context()
if !skipCP {
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
return err
}
}
return upgradeNodes(ctx, out, client, v, nodes, perNode)
},
}
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
return c
}
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
if version.Version == v {
fmt.Fprintf(out, "this machine already runs %s\n", v)
return nil
}
if os.Geteuid() != 0 {
return errors.New("replacing this machine's binary needs root: run it with sudo, " +
"or pass --skip-control-plane to upgrade only the nodes")
}
self, err := os.Executable()
if err != nil {
return err
}
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
if err != nil {
return err
}
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
var before []adminapi.NodeView
_ = client().get(ctx, "/api/v1/nodes", &before)
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
}
fmt.Fprintf(out, " %s restarted; waiting for its nodes to reconnect\n", svc)
// Back when the admin API answers and every node that was ready is ready
// again: agents reconnect within seconds of the control plane returning.
want := 0
for _, n := range before {
if n.State == "ready" {
want++
}
}
deadline := time.Now().Add(2 * time.Minute)
for {
var now []adminapi.NodeView
if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
ready := 0
for _, n := range now {
if n.State == "ready" {
ready++
}
}
if ready >= want {
fmt.Fprintf(out, " back: %d of %d nodes ready\n", ready, len(now))
return nil
}
}
if time.Now().After(deadline) {
return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
}
time.Sleep(2 * time.Second)
}
}
func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
var nodes []adminapi.NodeView
if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
return err
}
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
done, skipped := 0, 0
for _, n := range nodes {
if len(only) > 0 && !slices.Contains(only, n.ID) {
continue
}
if n.AgentVersion == v {
skipped++
continue
}
start := time.Now()
before := readyReplicasOn(ctx, client, n.ID)
fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
}
fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before)
deadline := time.Now().Add(perNode)
for {
var now []adminapi.NodeView
_ = client().get(ctx, "/api/v1/nodes", &now)
var cur adminapi.NodeView
for _, x := range now {
if x.ID == n.ID {
cur = x
}
}
after := readyReplicasOn(ctx, client, n.ID)
if cur.AgentVersion == v && cur.State == "ready" && after >= before {
fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n",
time.Since(start).Round(time.Second), after, before)
done++
break
}
if time.Now().After(deadline) {
return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
cur.State, after, before, n.ID)
}
time.Sleep(2 * time.Second)
}
}
fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
return nil
}
// readyReplicasOn counts the ready replicas with a shard on a node.
func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
var reps []adminapi.ReplicaView
if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
return 0
}
n := 0
for _, r := range reps {
if r.State != "ready" {
continue
}
for _, s := range r.Shards {
if s.Node == node {
n++
break
}
}
}
return n
}
+88
View File
@@ -0,0 +1,88 @@
package cli
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// fakeFleet is an admin API whose nodes come back on the version they are told
// to upgrade to — except those listed in stuck.
type fakeFleet struct {
mu sync.Mutex
versions map[string]string
stuck map[string]bool
told []string
}
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
switch {
case r.URL.Path == "/api/v1/nodes":
var out []adminapi.NodeView
for id, v := range f.versions {
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
}
_ = json.NewEncoder(w).Encode(out)
case r.URL.Path == "/api/v1/replicas":
var out []adminapi.ReplicaView
for id := range f.versions {
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
Shards: []adminapi.ShardView{{Node: id}}})
}
_ = json.NewEncoder(w).Encode(out)
case strings.HasSuffix(r.URL.Path, "/upgrade"):
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
var req adminapi.UpgradeRequest
_ = json.NewDecoder(r.Body).Decode(&req)
f.told = append(f.told, id)
if !f.stuck[id] {
f.versions[id] = req.Version
}
_, _ = w.Write([]byte(`{}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv
}
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
srv := f.server(t)
var out bytes.Buffer
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
if err != nil {
t.Fatal(err)
}
if strings.Join(f.told, ",") != "a,c" {
t.Errorf("told %v; b was already on the version", f.told)
}
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
t.Errorf("output:\n%s", out.String())
}
}
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
// A bad release reaches one node, not all of them.
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
srv := f.server(t)
var out bytes.Buffer
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
t.Fatalf("err %v", err)
}
if len(f.told) != 1 {
t.Errorf("went on to %v after a node failed to come back", f.told)
}
}
+14
View File
@@ -173,6 +173,7 @@ func newServerCmd() *cobra.Command {
}
if admin != nil {
api := adminapi.New(admin)
api.SetReleases(cfg.Releases)
// Catalog changes need somewhere to persist, so this is wired
// only when there is a store. Without one, the endpoints say so
// rather than accepting a model that vanishes on restart.
@@ -546,6 +547,19 @@ func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmg
return ft.FabricTest(ctx, nodeID, req)
}
// Upgrade forwards a node upgrade, as Logs does.
func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error {
d, err := r.get()
if err != nil {
return err
}
u, ok := d.(control.Upgrader)
if !ok {
return errors.New("this dispatcher cannot upgrade nodes")
}
return u.Upgrade(ctx, nodeID, version, releases)
}
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
d, err := r.get()
if err != nil {
+5
View File
@@ -86,6 +86,11 @@ type Server struct {
// Empty means /var/backups/llmgw-server.
BackupDir string `koanf:"backup_dir"`
// Releases is where `llmgw ctl upgrade` fetches release binaries from,
// for this machine and — through their agents — every node. Empty means
// the project's own releases.
Releases string `koanf:"releases"`
Router Router `koanf:"router"`
Control Control `koanf:"control"`
Models []catalog.Model `koanf:"models"`
+39
View File
@@ -10,6 +10,7 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"net/http"
"slices"
@@ -30,6 +31,9 @@ import (
// API serves the management endpoints.
type API struct {
// releases is where release binaries are fetched from, for upgrades.
releases string
models Models
capacity Capacity
usage Usage
@@ -74,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
@@ -1165,3 +1170,37 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
})
}
// SetReleases sets where upgrades fetch release binaries from.
func (a *API) SetReleases(base string) { a.releases = base }
// UpgradeRequest names the release a node should run.
type UpgradeRequest struct {
Version string `json:"version"`
}
func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
var req UpgradeRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error())
return
}
if !release.Tag.MatchString(req.Version) {
writeError(w, http.StatusBadRequest, req.Version+" is not a release version")
return
}
u := a.ctl.Upgrader()
if u == nil {
writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes")
return
}
base := a.releases
if base == "" {
base = release.DefaultBase
}
if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil {
writeError(w, http.StatusBadGateway, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
}
@@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
t.Errorf("imex = %+v", n.IMEX)
}
}
func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
s := newTestService()
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
s.conns["n1"] = c
done := make(chan error, 1)
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
req := (<-c.send).GetUpgrade()
if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" {
t.Fatalf("sent %v", req)
}
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{
RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch",
}}})
if err := <-done; err == nil || err.Error() != "checksum mismatch" {
t.Errorf("the agent's refusal was not passed on: %v", err)
}
}
+52
View File
@@ -60,6 +60,8 @@ type Service struct {
logMu sync.Mutex
logWaits map[string]chan *llmgwv1.LogsReply
logSeq uint64
// upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu.
upgradeWaits map[string]chan *llmgwv1.UpgradeReply
// scanWaits are the fabric scans awaiting a reply, by request id.
scanWaits map[string]chan *llmgwv1.FabricScanReply
testWaits map[string]chan *llmgwv1.FabricTestReply
@@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
ch <- m.FabricScan
}
case *llmgwv1.AgentMessage_Upgrade:
s.logMu.Lock()
ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()]
delete(s.upgradeWaits, m.Upgrade.GetRequestId())
s.logMu.Unlock()
if ok {
ch <- m.Upgrade
}
case *llmgwv1.AgentMessage_Logs:
s.logMu.Lock()
ch, ok := s.logWaits[m.Logs.GetRequestId()]
@@ -842,3 +853,44 @@ func installable(reg *llmgwv1.Register) []string {
// The admin API finds the bandwidth test by asserting this; a signature that
// drifted would make it quietly nil.
var _ fabrictest.Asker = (*Service)(nil)
// Upgrade tells a node's agent to replace its binary with a release and
// restart, and returns once the agent says the new binary is in place — or
// why it is not. The agent then exits and comes back on the new version; the
// caller watches for that in the inventory.
func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error {
s.logMu.Lock()
if s.upgradeWaits == nil {
s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{}
}
s.logSeq++
id := fmt.Sprintf("upgrade-%d", s.logSeq)
ch := make(chan *llmgwv1.UpgradeReply, 1)
s.upgradeWaits[id] = ch
s.logMu.Unlock()
defer func() {
s.logMu.Lock()
delete(s.upgradeWaits, id)
s.logMu.Unlock()
}()
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{
Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases},
}}); err != nil {
return err
}
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
// A download of some tens of megabytes on a slow link, and no longer: an
// agent too old to know this message never answers.
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
select {
case r := <-ch:
if r.GetError() != "" {
return errors.New(r.GetError())
}
return nil
case <-ctx.Done():
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
}
}
+11
View File
@@ -211,6 +211,17 @@ func (c *Controller) WeightRemover() WeightRemover {
return r
}
// Upgrader can tell a node's agent to upgrade itself.
type Upgrader interface {
Upgrade(ctx context.Context, nodeID, version, releases string) error
}
// Upgrader returns the dispatcher's upgrade capability, or nil.
func (c *Controller) Upgrader() Upgrader {
u, _ := c.disp.(Upgrader)
return u
}
// LogReader can ask a node for the recent lines of one of its logs.
type LogReader interface {
Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error)
+2 -1
View File
@@ -18,6 +18,7 @@ import (
"encoding/pem"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"net"
"os"
"path/filepath"
@@ -335,7 +336,7 @@ func certNames(path string) ([]string, error) {
// releaseTag is a version CI published: v0.10.4, or a release candidate of
// it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no
// release to download.
var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
var releaseTag = release.Tag
func (r *Runner) binary(ctx context.Context, res *Result) error {
v := r.Opts.Version
+148
View File
@@ -0,0 +1,148 @@
// Package release fetches and installs llmgw's own release binaries.
//
// One implementation for everywhere a binary is replaced — the control plane
// upgrading itself, and an agent told to upgrade by it — so the checks are
// the same on both: the release's own checksum is verified before anything is
// installed, the old binary is kept beside the new one, and the swap is a
// rename, so a machine is never left without a binary that runs.
package release
import (
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
// DefaultBase is where CI publishes releases.
const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download"
// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2.
var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
// AssetName is the release file for a platform.
func AssetName(version, goos, goarch string) string {
return "llmgw-" + version + "-" + goos + "-" + goarch
}
// Install fetches version for this platform from base, checks it against the
// release's checksums, and puts it at target, keeping the old one as
// target.bak-pre-<version>. It returns the backup's path.
func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) {
if !Tag.MatchString(version) {
return "", fmt.Errorf("%q is not a release version", version)
}
if client == nil {
client = &http.Client{Timeout: 10 * time.Minute}
}
target, err := filepath.EvalSymlinks(target)
if err != nil {
return "", err
}
dir := filepath.Dir(target)
asset := AssetName(version, goos, goarch)
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
want, err := checksum(ctx, client, root+"checksums.txt", asset)
if err != nil {
return "", err
}
// Beside the target, so the final rename is on one filesystem and atomic.
tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*")
if err != nil {
return "", err
}
defer os.Remove(tmp.Name())
got, err := download(ctx, client, root+asset, tmp)
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err != nil {
return "", err
}
if got != want {
return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
asset, got[:12], want[:12])
}
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
return "", err
}
backup := target + ".bak-pre-" + version
if err := copyFile(target, backup); err != nil {
return "", fmt.Errorf("keeping the old binary: %w", err)
}
if err := os.Rename(tmp.Name(), target); err != nil {
return "", err
}
return backup, nil
}
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
body, err := get(ctx, client, url)
if err != nil {
return "", err
}
defer body.Close()
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
for sc.Scan() {
f := strings.Fields(sc.Text())
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
return f[0], nil
}
}
return "", fmt.Errorf("the release's checksums list no %s", asset)
}
func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) {
body, err := get(ctx, client, url)
if err != nil {
return "", err
}
defer body.Close()
h := sha256.New()
if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil {
return "", fmt.Errorf("downloading %s: %w", url, err)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("fetching %s: %w", url, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("fetching %s: %s", url, resp.Status)
}
return resp.Body, nil
}
func copyFile(src, dst string) error {
in, err := os.Open(src) //nolint:gosec // our own binary
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
return err
}
return out.Close()
}
+61
View File
@@ -0,0 +1,61 @@
package release
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
)
func releaseServer(t *testing.T, version, asset string, body []byte, sum string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/" + version + "/checksums.txt":
_, _ = w.Write([]byte(sum + " " + asset + "\n"))
case "/" + version + "/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv
}
func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "llmgw")
_ = os.WriteFile(target, []byte("old"), 0o755)
body := []byte("new binary")
sum := sha256.Sum256(body)
asset := AssetName("v1.2.3", "linux", "amd64")
srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:]))
backup, err := Install(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "amd64", target)
if err != nil {
t.Fatal(err)
}
if b, _ := os.ReadFile(target); string(b) != "new binary" {
t.Errorf("target holds %q", b)
}
if b, _ := os.ReadFile(backup); string(b) != "old" {
t.Errorf("the old binary was not kept: %q", b)
}
// A download that does not match its checksum changes nothing.
bad := releaseServer(t, "v1.2.4", AssetName("v1.2.4", "linux", "amd64"), []byte("tampered"),
hex.EncodeToString(sum[:]))
if _, err := Install(t.Context(), bad.Client(), bad.URL, "v1.2.4", "linux", "amd64", target); err == nil {
t.Fatal("a binary not matching its checksum was installed")
}
if b, _ := os.ReadFile(target); string(b) != "new binary" {
t.Errorf("a failed install touched the binary: %q", b)
}
if _, err := Install(t.Context(), nil, srv.URL, "../../x", "linux", "amd64", target); err == nil {
t.Error("a version that is not one was fetched")
}
}