Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
@@ -77,6 +77,17 @@ more than 26 hours old or there has never been one. docs/install.md now says how
|
||||
to restore, including why running engines survive it: the engine-key secret
|
||||
comes back with the store.
|
||||
|
||||
### Upgrading from one command
|
||||
|
||||
`sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that
|
||||
machine and then every node in turn. Each agent is told the version over its
|
||||
existing stream, fetches the release for its own platform, checks it against
|
||||
the release's checksums, keeps the old binary beside the new and restarts,
|
||||
while its engines keep serving. A node that does not come back on the new
|
||||
version, ready, with its replicas, stops the upgrade before the next one.
|
||||
Running it again resumes. Agents from before this have to be moved by hand
|
||||
once.
|
||||
|
||||
### Following a request
|
||||
|
||||
Every response carries `X-Request-Id` — the client's own when it sends a plain
|
||||
|
||||
+241
-57
@@ -104,6 +104,7 @@ type AgentMessage struct {
|
||||
// *AgentMessage_Logs
|
||||
// *AgentMessage_FabricScan
|
||||
// *AgentMessage_FabricTest
|
||||
// *AgentMessage_Upgrade
|
||||
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -200,6 +201,15 @@ func (x *AgentMessage) GetFabricTest() *FabricTestReply {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AgentMessage) GetUpgrade() *UpgradeReply {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*AgentMessage_Upgrade); ok {
|
||||
return x.Upgrade
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isAgentMessage_Msg interface {
|
||||
isAgentMessage_Msg()
|
||||
}
|
||||
@@ -228,6 +238,10 @@ type AgentMessage_FabricTest struct {
|
||||
FabricTest *FabricTestReply `protobuf:"bytes,6,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
|
||||
}
|
||||
|
||||
type AgentMessage_Upgrade struct {
|
||||
Upgrade *UpgradeReply `protobuf:"bytes,7,opt,name=upgrade,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*AgentMessage_Register) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
|
||||
@@ -240,6 +254,8 @@ func (*AgentMessage_FabricScan) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_FabricTest) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_Upgrade) isAgentMessage_Msg() {}
|
||||
|
||||
// ControlMessage is anything the control plane tells a node.
|
||||
type ControlMessage struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -254,6 +270,7 @@ type ControlMessage struct {
|
||||
// *ControlMessage_Logs
|
||||
// *ControlMessage_FabricScan
|
||||
// *ControlMessage_FabricTest
|
||||
// *ControlMessage_Upgrade
|
||||
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -377,6 +394,15 @@ func (x *ControlMessage) GetFabricTest() *FabricTestRequest {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetUpgrade() *UpgradeAgent {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*ControlMessage_Upgrade); ok {
|
||||
return x.Upgrade
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isControlMessage_Msg interface {
|
||||
isControlMessage_Msg()
|
||||
}
|
||||
@@ -417,6 +443,10 @@ type ControlMessage_FabricTest struct {
|
||||
FabricTest *FabricTestRequest `protobuf:"bytes,9,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
|
||||
}
|
||||
|
||||
type ControlMessage_Upgrade struct {
|
||||
Upgrade *UpgradeAgent `protobuf:"bytes,10,opt,name=upgrade,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*ControlMessage_Start) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_Stop) isControlMessage_Msg() {}
|
||||
@@ -435,6 +465,8 @@ func (*ControlMessage_FabricScan) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_FabricTest) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_Upgrade) isControlMessage_Msg() {}
|
||||
|
||||
// Register is the first message on every stream. It carries the node's full
|
||||
// hardware inventory, so the control plane needs no separate discovery call and
|
||||
// a reconnecting agent automatically refreshes what the scheduler knows.
|
||||
@@ -3460,11 +3492,144 @@ func (x *FabricTestReply) GetTookMs() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// UpgradeAgent tells an agent to replace its own binary with a release and
|
||||
// restart.
|
||||
//
|
||||
// The agent fetches the release for its own platform and checks it against
|
||||
// the release's checksums before replacing anything, keeps the old binary
|
||||
// beside the new, then exits so systemd starts the new one. Engines keep
|
||||
// running throughout, as for any agent restart, and are adopted on the way
|
||||
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
||||
// no ssh into it.
|
||||
type UpgradeAgent struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
||||
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
|
||||
// releases is the base URL release files are downloaded from.
|
||||
Releases string `protobuf:"bytes,3,opt,name=releases,proto3" json:"releases,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) Reset() {
|
||||
*x = UpgradeAgent{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UpgradeAgent) ProtoMessage() {}
|
||||
|
||||
func (x *UpgradeAgent) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead.
|
||||
func (*UpgradeAgent) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33}
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) GetVersion() string {
|
||||
if x != nil {
|
||||
return x.Version
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeAgent) GetReleases() string {
|
||||
if x != nil {
|
||||
return x.Releases
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// UpgradeReply says whether the new binary is in place. Sent before the
|
||||
// agent exits; an empty error means it will be back on the new version.
|
||||
type UpgradeReply struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
|
||||
Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) Reset() {
|
||||
*x = UpgradeReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UpgradeReply) ProtoMessage() {}
|
||||
|
||||
func (x *UpgradeReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead.
|
||||
func (*UpgradeReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34}
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) GetVersion() string {
|
||||
if x != nil {
|
||||
return x.Version
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpgradeReply) GetError() string {
|
||||
if x != nil {
|
||||
return x.Error
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd2\x02\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x86\x03\n" +
|
||||
"\fAgentMessage\x120\n" +
|
||||
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
|
||||
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
|
||||
@@ -3473,8 +3638,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" +
|
||||
"fabricScan\x12<\n" +
|
||||
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
|
||||
"fabricTestB\x05\n" +
|
||||
"\x03msg\"\xe5\x03\n" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgradeB\x05\n" +
|
||||
"\x03msg\"\x99\x04\n" +
|
||||
"\x0eControlMessage\x12,\n" +
|
||||
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
@@ -3486,7 +3652,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
|
||||
"fabricScan\x12>\n" +
|
||||
"\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" +
|
||||
"fabricTestB\x05\n" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\n" +
|
||||
" \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" +
|
||||
"\x03msg\"\xe3\x06\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
@@ -3795,7 +3963,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04gbps\x18\x02 \x01(\x01R\x04gbps\x12\x16\n" +
|
||||
"\x06output\x18\x03 \x01(\tR\x06output\x12\x14\n" +
|
||||
"\x05error\x18\x04 \x01(\tR\x05error\x12\x17\n" +
|
||||
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs2O\n" +
|
||||
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs\"c\n" +
|
||||
"\fUpgradeAgent\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x1a\n" +
|
||||
"\breleases\x18\x03 \x01(\tR\breleases\"]\n" +
|
||||
"\fUpgradeReply\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -3814,7 +3992,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 41)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -3850,12 +4028,14 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
|
||||
(*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest
|
||||
(*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply
|
||||
nil, // 34: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 35: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 36: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 37: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 38: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 39: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*UpgradeAgent)(nil), // 34: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 35: llmgw.v1.UpgradeReply
|
||||
nil, // 36: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 37: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 38: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 39: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 40: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 41: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -3864,50 +4044,52 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
|
||||
33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
|
||||
22, // 6: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 7: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 8: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 9: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 10: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 11: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 12: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 13: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 14: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
34, // 15: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 16: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 17: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
35, // 18: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 19: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 20: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 21: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 22: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
14, // 23: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 24: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
36, // 25: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 26: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
37, // 27: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 28: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 29: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 30: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
38, // 31: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 32: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 33: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 34: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 35: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 36: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 37: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 38: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 39: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 40: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 41: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
39, // 42: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 43: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 44: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
44, // [44:45] is the sub-list for method output_type
|
||||
43, // [43:44] is the sub-list for method input_type
|
||||
43, // [43:43] is the sub-list for extension type_name
|
||||
43, // [43:43] is the sub-list for extension extendee
|
||||
0, // [0:43] is the sub-list for field type_name
|
||||
35, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply
|
||||
22, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
34, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
36, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
37, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
14, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
38, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
39, // 29: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 30: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 31: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 32: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
40, // 33: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 34: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 35: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 36: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 37: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 38: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 39: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 40: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 41: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 42: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 43: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
41, // 44: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 45: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 46: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
46, // [46:47] is the sub-list for method output_type
|
||||
45, // [45:46] is the sub-list for method input_type
|
||||
45, // [45:45] is the sub-list for extension type_name
|
||||
45, // [45:45] is the sub-list for extension extendee
|
||||
0, // [0:45] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -3922,6 +4104,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*AgentMessage_Logs)(nil),
|
||||
(*AgentMessage_FabricScan)(nil),
|
||||
(*AgentMessage_FabricTest)(nil),
|
||||
(*AgentMessage_Upgrade)(nil),
|
||||
}
|
||||
file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{
|
||||
(*ControlMessage_Start)(nil),
|
||||
@@ -3933,6 +4116,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*ControlMessage_Logs)(nil),
|
||||
(*ControlMessage_FabricScan)(nil),
|
||||
(*ControlMessage_FabricTest)(nil),
|
||||
(*ControlMessage_Upgrade)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@@ -3940,7 +4124,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 39,
|
||||
NumMessages: 41,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -26,6 +26,7 @@ message AgentMessage {
|
||||
LogsReply logs = 4;
|
||||
FabricScanReply fabric_scan = 5;
|
||||
FabricTestReply fabric_test = 6;
|
||||
UpgradeReply upgrade = 7;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +42,7 @@ message ControlMessage {
|
||||
LogsRequest logs = 7;
|
||||
FabricScanRequest fabric_scan = 8;
|
||||
FabricTestRequest fabric_test = 9;
|
||||
UpgradeAgent upgrade = 10;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -649,3 +651,28 @@ message FabricTestReply {
|
||||
string error = 4;
|
||||
int64 took_ms = 5;
|
||||
}
|
||||
|
||||
// UpgradeAgent tells an agent to replace its own binary with a release and
|
||||
// restart.
|
||||
//
|
||||
// The agent fetches the release for its own platform and checks it against
|
||||
// the release's checksums before replacing anything, keeps the old binary
|
||||
// beside the new, then exits so systemd starts the new one. Engines keep
|
||||
// running throughout, as for any agent restart, and are adopted on the way
|
||||
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
||||
// no ssh into it.
|
||||
message UpgradeAgent {
|
||||
string request_id = 1;
|
||||
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
||||
string version = 2;
|
||||
// releases is the base URL release files are downloaded from.
|
||||
string releases = 3;
|
||||
}
|
||||
|
||||
// UpgradeReply says whether the new binary is in place. Sent before the
|
||||
// agent exits; an empty error means it will be back on the new version.
|
||||
message UpgradeReply {
|
||||
string request_id = 1;
|
||||
string version = 2;
|
||||
string error = 3;
|
||||
}
|
||||
+29
-32
@@ -909,55 +909,52 @@ refuses every request.
|
||||
|
||||
## Upgrading
|
||||
|
||||
One binary on every machine, so an upgrade is a copy and a restart. The part
|
||||
worth care is the copy, not the order.
|
||||
On the control plane:
|
||||
|
||||
```
|
||||
sudo llmgw ctl upgrade v0.10.4
|
||||
```
|
||||
|
||||
It upgrades this machine first — fetching the release for its platform,
|
||||
checking it against the release's checksums, keeping the old binary as
|
||||
`/usr/local/bin/llmgw.bak-pre-<version>`, restarting `llmgw-server` and waiting
|
||||
for every node to reconnect — and then each node in turn. A node's agent is told
|
||||
the version over the stream it already holds, fetches and checks the release
|
||||
itself, swaps it in and restarts; no ssh into the nodes is needed. Release
|
||||
candidates (`v0.10.4-rc.2`) work the same way.
|
||||
|
||||
A node counts as done when it is back on the new version, ready, with as many
|
||||
ready replicas as before. One that is not within `--timeout` (three minutes)
|
||||
stops the upgrade there, so a bad release reaches one node rather than all of
|
||||
them; `llmgw ctl logs <node>` says why, and the node keeps its previous binary
|
||||
beside the new one. Nodes already on the version are skipped, so running the
|
||||
command again resumes where it stopped. `--nodes a,b` limits it to some nodes,
|
||||
and `--skip-control-plane` leaves this machine alone.
|
||||
|
||||
**Nothing stops serving.** Engines are started with `KillMode=process`, so they
|
||||
keep running when the agent restarts, and the agent reports them back on its
|
||||
next connection. The control plane holds no replica state across its own
|
||||
restart either — it rebuilds the fleet from what the agents report. A rolling
|
||||
upgrade of the whole cluster interrupts no request that is not in flight at the
|
||||
moment of the restart.
|
||||
restart either — it rebuilds the fleet from what the agents report.
|
||||
|
||||
**A mixed fleet is fine while you roll.** The agent protocol only ever gains
|
||||
fields, and both sides treat an absent one as unknown rather than as a default.
|
||||
Order does not matter; finish the roll the same day anyway, because
|
||||
`ctl nodes` reports each agent's version and a fleet that disagrees with itself
|
||||
is one more thing to hold in your head.
|
||||
|
||||
```
|
||||
VERSION=v0.9.0
|
||||
# copy to every machine first, install nowhere yet
|
||||
for h in 10.0.0.140 10.0.0.141 10.0.0.142 10.0.0.146; do
|
||||
ssh $h 'cat > /tmp/llmgw-new; stat -c %s /tmp/llmgw-new' < llmgw-$VERSION-linux-amd64
|
||||
done
|
||||
```
|
||||
|
||||
**Check every size before installing anything.** This is not defensive writing:
|
||||
on a host whose login shell prints anything, `scp` fails outright, and the
|
||||
obvious workaround — `ssh -n host 'cat > file'` — points stdin at `/dev/null`
|
||||
and writes a *zero-byte file* while reporting success. Installing that on a
|
||||
fleet takes the fleet down. Compare the number on each machine with the local
|
||||
one, then:
|
||||
|
||||
```
|
||||
sudo install -m0755 /tmp/llmgw-new /usr/local/bin/llmgw
|
||||
sudo systemctl restart llmgw-agent # or llmgw-server, or llmgw-mirror
|
||||
llmgw --version
|
||||
```
|
||||
Agents older than v0.10.4 cannot upgrade themselves: move those by hand once,
|
||||
installing the release binary and restarting `llmgw-agent`, and every upgrade
|
||||
after that is the command above. The mirror, which is not an agent, is
|
||||
upgraded the same way by hand.
|
||||
|
||||
Then confirm what you have, rather than assuming:
|
||||
|
||||
```
|
||||
llmgw ctl nodes # every agent back, LAST SEEN in seconds
|
||||
llmgw ctl nodes # every agent on the new version, LAST SEEN in seconds
|
||||
llmgw ctl replicas # same replicas, adopted rather than replaced
|
||||
```
|
||||
|
||||
A replica may come back marked **`ready (stale profile)`**. That means it is
|
||||
running a profile that has since been edited — the reconciler leaves a serving
|
||||
replica alone on purpose, because churning one costs more than the difference.
|
||||
Restart it when you want the new profile to take effect; the flag is there so
|
||||
the choice is yours rather than silent.
|
||||
`llmgw ctl restart <replica>` replaces it with one on the new profile, starting
|
||||
the new one before stopping the old.
|
||||
|
||||
## Backing up
|
||||
|
||||
|
||||
@@ -170,6 +170,7 @@ rather than repeated.
|
||||
| `mirror` | `Mirror` | [Server](#server) |
|
||||
| `engines` | `map[string]string` | [Server](#server) |
|
||||
| `backup_dir` | `string` | [Server](#server) |
|
||||
| `releases` | `string` | [Server](#server) |
|
||||
| `router` | `Router` | [Server](#server) |
|
||||
| `control` | `Control` | [Server](#server) |
|
||||
| `models` | `[]catalog.Model` | [Server](#server) |
|
||||
@@ -1274,6 +1275,14 @@ BackupDir is where `llmgw server backup` writes, and where the newest
|
||||
archive's age is read for llmgw_backup_last_success_timestamp_seconds.
|
||||
Empty means /var/backups/llmgw-server.
|
||||
|
||||
### `releases`
|
||||
|
||||
`string`
|
||||
|
||||
Releases is where `llmgw ctl upgrade` fetches release binaries from,
|
||||
for this machine and — through their agents — every node. Empty means
|
||||
the project's own releases.
|
||||
|
||||
### `router`
|
||||
|
||||
`Router`
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -1119,6 +1121,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||
case *llmgwv1.ControlMessage_Provision:
|
||||
a.setProvision(m.Provision)
|
||||
case *llmgwv1.ControlMessage_Upgrade:
|
||||
go a.upgrade(ctx, m.Upgrade, out)
|
||||
case *llmgwv1.ControlMessage_Logs:
|
||||
reply := a.logsReply(m.Logs)
|
||||
// Off the receive loop: a full send queue must not stop commands being
|
||||
@@ -1874,3 +1878,49 @@ func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||
m := discoverIMEX(ctx)
|
||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||
}
|
||||
|
||||
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
|
||||
// Restart=on-failure starts the new binary, and distinct, so the journal says
|
||||
// why the agent went.
|
||||
const upgradeExit = 75
|
||||
|
||||
// exitForUpgrade ends the process; a variable so a test can see it called
|
||||
// rather than have its own process end.
|
||||
var exitForUpgrade = func() { os.Exit(upgradeExit) }
|
||||
|
||||
// upgradeTarget is the binary an upgrade replaces; a variable so a test
|
||||
// replaces a file of its own rather than the test binary running it.
|
||||
var upgradeTarget = os.Executable
|
||||
|
||||
// upgrade replaces this agent's binary with a release and restarts into it.
|
||||
//
|
||||
// Engines are not touched: an agent restart leaves them running and adopts
|
||||
// them again, which is what makes upgrading a node something that can be done
|
||||
// while it serves.
|
||||
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
|
||||
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
|
||||
send := func() {
|
||||
select {
|
||||
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
|
||||
self, err := upgradeTarget()
|
||||
if err == nil {
|
||||
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
|
||||
cancel()
|
||||
}
|
||||
if err != nil {
|
||||
reply.Error = err.Error()
|
||||
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
|
||||
send()
|
||||
return
|
||||
}
|
||||
send()
|
||||
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
|
||||
// A moment for the reply to leave before the stream goes.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
)
|
||||
|
||||
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
|
||||
body := []byte("the new agent")
|
||||
sum := sha256.Sum256(body)
|
||||
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v9.9.9/checksums.txt":
|
||||
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
|
||||
case "/v9.9.9/" + asset:
|
||||
_, _ = w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
bin := filepath.Join(t.TempDir(), "llmgw")
|
||||
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
|
||||
exited := make(chan struct{}, 1)
|
||||
oldTarget, oldExit := upgradeTarget, exitForUpgrade
|
||||
upgradeTarget = func() (string, error) { return bin, nil }
|
||||
exitForUpgrade = func() { exited <- struct{}{} }
|
||||
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
|
||||
|
||||
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
|
||||
cfg.Superv.WorkDir = t.TempDir()
|
||||
a, err := New(cfg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := make(chan *llmgwv1.AgentMessage, 1)
|
||||
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
|
||||
|
||||
select {
|
||||
case m := <-out:
|
||||
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
|
||||
t.Fatalf("reply %+v", r)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no reply")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the agent did not restart into the new binary")
|
||||
}
|
||||
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
|
||||
t.Errorf("binary holds %q", b)
|
||||
}
|
||||
|
||||
// A version with no release answers with why, and does not restart.
|
||||
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
|
||||
if r := (<-out).GetUpgrade(); r.GetError() == "" {
|
||||
t.Error("an upgrade to a release that does not exist reported success")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
t.Error("a failed upgrade restarted the agent")
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newScaleCmd(client),
|
||||
newRestartCmd(client),
|
||||
newRetryCmd(client),
|
||||
newUpgradeCmd(client),
|
||||
newEngineCheckCmd(),
|
||||
newToolCheckCmd(),
|
||||
newWeightsCmd(client, &asJSON),
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
nodes []string
|
||||
skipCP bool
|
||||
releases string
|
||||
perNode time.Duration
|
||||
serverSvc string
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "upgrade VERSION",
|
||||
Short: "Upgrade this control plane, then every node, one at a time",
|
||||
Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
|
||||
"This machine first — run it with sudo on the control plane, since it\n" +
|
||||
"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
|
||||
"turn: its agent fetches the release for its own platform, checks it\n" +
|
||||
"against the release's checksums, swaps it in and restarts. Engines keep\n" +
|
||||
"serving through all of it and are adopted again; no ssh into the nodes is\n" +
|
||||
"needed.\n\n" +
|
||||
"A node counts as done when it is back on the new version, ready, with as\n" +
|
||||
"many ready replicas as before. One that is not within --timeout stops the\n" +
|
||||
"upgrade there, so a bad release reaches one node rather than all of them.\n" +
|
||||
"Nodes already on the version are skipped, so running it again resumes.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
v := args[0]
|
||||
if !release.Tag.MatchString(v) {
|
||||
return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
ctx := cmd.Context()
|
||||
if !skipCP {
|
||||
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return upgradeNodes(ctx, out, client, v, nodes, perNode)
|
||||
},
|
||||
}
|
||||
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
|
||||
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
|
||||
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
|
||||
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
|
||||
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
|
||||
return c
|
||||
}
|
||||
|
||||
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
|
||||
if version.Version == v {
|
||||
fmt.Fprintf(out, "this machine already runs %s\n", v)
|
||||
return nil
|
||||
}
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("replacing this machine's binary needs root: run it with sudo, " +
|
||||
"or pass --skip-control-plane to upgrade only the nodes")
|
||||
}
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
|
||||
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
|
||||
var before []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &before)
|
||||
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
|
||||
return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
|
||||
}
|
||||
fmt.Fprintf(out, " %s restarted; waiting for its nodes to reconnect\n", svc)
|
||||
// Back when the admin API answers and every node that was ready is ready
|
||||
// again: agents reconnect within seconds of the control plane returning.
|
||||
want := 0
|
||||
for _, n := range before {
|
||||
if n.State == "ready" {
|
||||
want++
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Minute)
|
||||
for {
|
||||
var now []adminapi.NodeView
|
||||
if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
|
||||
ready := 0
|
||||
for _, n := range now {
|
||||
if n.State == "ready" {
|
||||
ready++
|
||||
}
|
||||
}
|
||||
if ready >= want {
|
||||
fmt.Fprintf(out, " back: %d of %d nodes ready\n", ready, len(now))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
|
||||
"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
|
||||
var nodes []adminapi.NodeView
|
||||
if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
|
||||
done, skipped := 0, 0
|
||||
for _, n := range nodes {
|
||||
if len(only) > 0 && !slices.Contains(only, n.ID) {
|
||||
continue
|
||||
}
|
||||
if n.AgentVersion == v {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
start := time.Now()
|
||||
before := readyReplicasOn(ctx, client, n.ID)
|
||||
fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
|
||||
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
|
||||
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||
return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
|
||||
}
|
||||
fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before)
|
||||
deadline := time.Now().Add(perNode)
|
||||
for {
|
||||
var now []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &now)
|
||||
var cur adminapi.NodeView
|
||||
for _, x := range now {
|
||||
if x.ID == n.ID {
|
||||
cur = x
|
||||
}
|
||||
}
|
||||
after := readyReplicasOn(ctx, client, n.ID)
|
||||
if cur.AgentVersion == v && cur.State == "ready" && after >= before {
|
||||
fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n",
|
||||
time.Since(start).Round(time.Second), after, before)
|
||||
done++
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
|
||||
"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
|
||||
"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
|
||||
cur.State, after, before, n.ID)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
|
||||
return nil
|
||||
}
|
||||
|
||||
// readyReplicasOn counts the ready replicas with a shard on a node.
|
||||
func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, r := range reps {
|
||||
if r.State != "ready" {
|
||||
continue
|
||||
}
|
||||
for _, s := range r.Shards {
|
||||
if s.Node == node {
|
||||
n++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// fakeFleet is an admin API whose nodes come back on the version they are told
|
||||
// to upgrade to — except those listed in stuck.
|
||||
type fakeFleet struct {
|
||||
mu sync.Mutex
|
||||
versions map[string]string
|
||||
stuck map[string]bool
|
||||
told []string
|
||||
}
|
||||
|
||||
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/nodes":
|
||||
var out []adminapi.NodeView
|
||||
for id, v := range f.versions {
|
||||
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case r.URL.Path == "/api/v1/replicas":
|
||||
var out []adminapi.ReplicaView
|
||||
for id := range f.versions {
|
||||
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
|
||||
Shards: []adminapi.ShardView{{Node: id}}})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case strings.HasSuffix(r.URL.Path, "/upgrade"):
|
||||
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
|
||||
var req adminapi.UpgradeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.told = append(f.told, id)
|
||||
if !f.stuck[id] {
|
||||
f.versions[id] = req.Version
|
||||
}
|
||||
_, _ = w.Write([]byte(`{}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(f.told, ",") != "a,c" {
|
||||
t.Errorf("told %v; b was already on the version", f.told)
|
||||
}
|
||||
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
|
||||
t.Errorf("output:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
|
||||
// A bad release reaches one node, not all of them.
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
|
||||
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
if len(f.told) != 1 {
|
||||
t.Errorf("went on to %v after a node failed to come back", f.told)
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,7 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
api.SetReleases(cfg.Releases)
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
// only when there is a store. Without one, the endpoints say so
|
||||
// rather than accepting a model that vanishes on restart.
|
||||
@@ -546,6 +547,19 @@ func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmg
|
||||
return ft.FabricTest(ctx, nodeID, req)
|
||||
}
|
||||
|
||||
// Upgrade forwards a node upgrade, as Logs does.
|
||||
func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u, ok := d.(control.Upgrader)
|
||||
if !ok {
|
||||
return errors.New("this dispatcher cannot upgrade nodes")
|
||||
}
|
||||
return u.Upgrade(ctx, nodeID, version, releases)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
|
||||
@@ -86,6 +86,11 @@ type Server struct {
|
||||
// Empty means /var/backups/llmgw-server.
|
||||
BackupDir string `koanf:"backup_dir"`
|
||||
|
||||
// Releases is where `llmgw ctl upgrade` fetches release binaries from,
|
||||
// for this machine and — through their agents — every node. Empty means
|
||||
// the project's own releases.
|
||||
Releases string `koanf:"releases"`
|
||||
|
||||
Router Router `koanf:"router"`
|
||||
Control Control `koanf:"control"`
|
||||
Models []catalog.Model `koanf:"models"`
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
@@ -30,6 +31,9 @@ import (
|
||||
|
||||
// API serves the management endpoints.
|
||||
type API struct {
|
||||
// releases is where release binaries are fetched from, for upgrades.
|
||||
releases string
|
||||
|
||||
models Models
|
||||
capacity Capacity
|
||||
usage Usage
|
||||
@@ -74,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
@@ -1165,3 +1170,37 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
})
|
||||
}
|
||||
|
||||
// SetReleases sets where upgrades fetch release binaries from.
|
||||
func (a *API) SetReleases(base string) { a.releases = base }
|
||||
|
||||
// UpgradeRequest names the release a node should run.
|
||||
type UpgradeRequest struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
|
||||
var req UpgradeRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !release.Tag.MatchString(req.Version) {
|
||||
writeError(w, http.StatusBadRequest, req.Version+" is not a release version")
|
||||
return
|
||||
}
|
||||
u := a.ctl.Upgrader()
|
||||
if u == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes")
|
||||
return
|
||||
}
|
||||
base := a.releases
|
||||
if base == "" {
|
||||
base = release.DefaultBase
|
||||
}
|
||||
if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil {
|
||||
writeError(w, http.StatusBadGateway, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
|
||||
}
|
||||
@@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
||||
t.Errorf("imex = %+v", n.IMEX)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
|
||||
s := newTestService()
|
||||
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
|
||||
s.conns["n1"] = c
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
|
||||
|
||||
req := (<-c.send).GetUpgrade()
|
||||
if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" {
|
||||
t.Fatalf("sent %v", req)
|
||||
}
|
||||
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{
|
||||
RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch",
|
||||
}}})
|
||||
if err := <-done; err == nil || err.Error() != "checksum mismatch" {
|
||||
t.Errorf("the agent's refusal was not passed on: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,8 @@ type Service struct {
|
||||
logMu sync.Mutex
|
||||
logWaits map[string]chan *llmgwv1.LogsReply
|
||||
logSeq uint64
|
||||
// upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu.
|
||||
upgradeWaits map[string]chan *llmgwv1.UpgradeReply
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
testWaits map[string]chan *llmgwv1.FabricTestReply
|
||||
@@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
ch <- m.FabricScan
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Upgrade:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()]
|
||||
delete(s.upgradeWaits, m.Upgrade.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.Upgrade
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Logs:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
||||
@@ -842,3 +853,44 @@ func installable(reg *llmgwv1.Register) []string {
|
||||
// The admin API finds the bandwidth test by asserting this; a signature that
|
||||
// drifted would make it quietly nil.
|
||||
var _ fabrictest.Asker = (*Service)(nil)
|
||||
|
||||
// Upgrade tells a node's agent to replace its binary with a release and
|
||||
// restart, and returns once the agent says the new binary is in place — or
|
||||
// why it is not. The agent then exits and comes back on the new version; the
|
||||
// caller watches for that in the inventory.
|
||||
func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||
s.logMu.Lock()
|
||||
if s.upgradeWaits == nil {
|
||||
s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("upgrade-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.UpgradeReply, 1)
|
||||
s.upgradeWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.upgradeWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{
|
||||
Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases},
|
||||
}}); err != nil {
|
||||
return err
|
||||
}
|
||||
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
|
||||
// A download of some tens of megabytes on a slow link, and no longer: an
|
||||
// agent too old to know this message never answers.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.GetError() != "" {
|
||||
return errors.New(r.GetError())
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
|
||||
}
|
||||
}
|
||||
@@ -211,6 +211,17 @@ func (c *Controller) WeightRemover() WeightRemover {
|
||||
return r
|
||||
}
|
||||
|
||||
// Upgrader can tell a node's agent to upgrade itself.
|
||||
type Upgrader interface {
|
||||
Upgrade(ctx context.Context, nodeID, version, releases string) error
|
||||
}
|
||||
|
||||
// Upgrader returns the dispatcher's upgrade capability, or nil.
|
||||
func (c *Controller) Upgrader() Upgrader {
|
||||
u, _ := c.disp.(Upgrader)
|
||||
return u
|
||||
}
|
||||
|
||||
// LogReader can ask a node for the recent lines of one of its logs.
|
||||
type LogReader interface {
|
||||
Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error)
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -335,7 +336,7 @@ func certNames(path string) ([]string, error) {
|
||||
// releaseTag is a version CI published: v0.10.4, or a release candidate of
|
||||
// it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no
|
||||
// release to download.
|
||||
var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
|
||||
var releaseTag = release.Tag
|
||||
|
||||
func (r *Runner) binary(ctx context.Context, res *Result) error {
|
||||
v := r.Opts.Version
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
// Package release fetches and installs llmgw's own release binaries.
|
||||
//
|
||||
// One implementation for everywhere a binary is replaced — the control plane
|
||||
// upgrading itself, and an agent told to upgrade by it — so the checks are
|
||||
// the same on both: the release's own checksum is verified before anything is
|
||||
// installed, the old binary is kept beside the new one, and the swap is a
|
||||
// rename, so a machine is never left without a binary that runs.
|
||||
package release
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultBase is where CI publishes releases.
|
||||
const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download"
|
||||
|
||||
// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2.
|
||||
var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
|
||||
|
||||
// AssetName is the release file for a platform.
|
||||
func AssetName(version, goos, goarch string) string {
|
||||
return "llmgw-" + version + "-" + goos + "-" + goarch
|
||||
}
|
||||
|
||||
// Install fetches version for this platform from base, checks it against the
|
||||
// release's checksums, and puts it at target, keeping the old one as
|
||||
// target.bak-pre-<version>. It returns the backup's path.
|
||||
func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) {
|
||||
if !Tag.MatchString(version) {
|
||||
return "", fmt.Errorf("%q is not a release version", version)
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Minute}
|
||||
}
|
||||
target, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dir := filepath.Dir(target)
|
||||
asset := AssetName(version, goos, goarch)
|
||||
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
|
||||
|
||||
want, err := checksum(ctx, client, root+"checksums.txt", asset)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Beside the target, so the final rename is on one filesystem and atomic.
|
||||
tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
got, err := download(ctx, client, root+asset, tmp)
|
||||
if cerr := tmp.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if got != want {
|
||||
return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
|
||||
asset, got[:12], want[:12])
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
backup := target + ".bak-pre-" + version
|
||||
if err := copyFile(target, backup); err != nil {
|
||||
return "", fmt.Errorf("keeping the old binary: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), target); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return backup, nil
|
||||
}
|
||||
|
||||
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
|
||||
body, err := get(ctx, client, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer body.Close()
|
||||
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
|
||||
for sc.Scan() {
|
||||
f := strings.Fields(sc.Text())
|
||||
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
|
||||
return f[0], nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("the release's checksums list no %s", asset)
|
||||
}
|
||||
|
||||
func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) {
|
||||
body, err := get(ctx, client, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer body.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil {
|
||||
return "", fmt.Errorf("downloading %s: %w", url, err)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching %s: %w", url, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("fetching %s: %s", url, resp.Status)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src) //nolint:gosec // our own binary
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func releaseServer(t *testing.T, version, asset string, body []byte, sum string) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/" + version + "/checksums.txt":
|
||||
_, _ = w.Write([]byte(sum + " " + asset + "\n"))
|
||||
case "/" + version + "/" + asset:
|
||||
_, _ = w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "llmgw")
|
||||
_ = os.WriteFile(target, []byte("old"), 0o755)
|
||||
body := []byte("new binary")
|
||||
sum := sha256.Sum256(body)
|
||||
asset := AssetName("v1.2.3", "linux", "amd64")
|
||||
|
||||
srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:]))
|
||||
backup, err := Install(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "amd64", target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _ := os.ReadFile(target); string(b) != "new binary" {
|
||||
t.Errorf("target holds %q", b)
|
||||
}
|
||||
if b, _ := os.ReadFile(backup); string(b) != "old" {
|
||||
t.Errorf("the old binary was not kept: %q", b)
|
||||
}
|
||||
|
||||
// A download that does not match its checksum changes nothing.
|
||||
bad := releaseServer(t, "v1.2.4", AssetName("v1.2.4", "linux", "amd64"), []byte("tampered"),
|
||||
hex.EncodeToString(sum[:]))
|
||||
if _, err := Install(t.Context(), bad.Client(), bad.URL, "v1.2.4", "linux", "amd64", target); err == nil {
|
||||
t.Fatal("a binary not matching its checksum was installed")
|
||||
}
|
||||
if b, _ := os.ReadFile(target); string(b) != "new binary" {
|
||||
t.Errorf("a failed install touched the binary: %q", b)
|
||||
}
|
||||
if _, err := Install(t.Context(), nil, srv.URL, "../../x", "linux", "amd64", target); err == nil {
|
||||
t.Error("a version that is not one was fetched")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user