From aa3efcebbe1b12beddc4f0ed9b62f9e291e97423 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 22:29:12 +0200 Subject: [PATCH] Add a standby control plane with server add, and draw the screens as windows server add gives another machine this llmgw, a certificate, the configuration and store address, lets it into the store through Patroni, and starts it; --vip keeps a floating address on the leader with keepalived, following a new /leader endpoint. Nodes learn every control plane from the leader and notice a dead one by HTTP/2 ping instead of after the kernel's fifteen minutes. The setup, console, node and init screens are drawn full screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 28 + api/gen/llmgw/v1/agent.pb.go | 482 ++++++++++-------- api/proto/llmgw/v1/agent.proto | 9 + go.mod | 2 +- go.sum | 2 + internal/agent/agent.go | 46 +- internal/agent/controlplanes.go | 77 +++ internal/agent/controlplanes_test.go | 41 ++ internal/cli/server.go | 20 +- internal/cli/serveradd.go | 459 +++++++++++++++++ internal/cli/tui_frame.go | 184 +++++++ internal/cli/tui_home.go | 57 ++- internal/cli/tui_init.go | 54 +- internal/cli/tui_node.go | 73 ++- internal/cli/tui_setup.go | 280 +++++----- internal/cli/tui_setup_test.go | 23 +- internal/control/agentsvc/controlplanes.go | 65 +++ internal/control/agentsvc/service.go | 10 + internal/pgha/config.go | 19 +- internal/pgha/hba.go | 91 ++++ internal/pgha/pgha_test.go | 72 +++ internal/pgha/status.go | 2 +- internal/router/handlers.go | 15 +- internal/router/hardware_leader_test.go | 43 ++ internal/router/router.go | 5 + internal/store/controlplanes.go | 44 ++ internal/store/copy.go | 1 + .../postgres/0016_control_planes.sql | 10 + .../migrations/sqlite/0016_control_planes.sql | 9 + internal/store/postgres_test.go | 2 +- internal/store/store_test.go | 25 + 31 files changed, 1856 insertions(+), 394 deletions(-) create mode 100644 internal/agent/controlplanes.go create mode 100644 internal/agent/controlplanes_test.go create mode 100644 internal/cli/serveradd.go create mode 100644 internal/cli/tui_frame.go create mode 100644 internal/control/agentsvc/controlplanes.go create mode 100644 internal/pgha/hba.go create mode 100644 internal/router/hardware_leader_test.go create mode 100644 internal/store/controlplanes.go create mode 100644 internal/store/migrations/postgres/0016_control_planes.sql create mode 100644 internal/store/migrations/sqlite/0016_control_planes.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 830ebe7..254a4d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,34 @@ and a figure the driver does not give is left out rather than shown as zero. each card and whose quota it counts against, for joining the GPU series to models and tenants. +### A second control plane with `llmgw server add`, and the screens as windows + +- **`sudo llmgw server add [user@]host [--vip 10.0.0.160/24]`**, run on a + control plane, makes another machine a standby: this llmgw copied over, a + certificate from the cluster's authority, the same configuration and store + address, the machine let into the store (for a store under Patroni, one + change through Patroni's API), and the service started. Every node is told + of it within half a minute and fails over to it untouched. `--vip` keeps a + floating address on whichever control plane leads, with keepalived on both. + Tried on new machines: switching the leader off, the standby led after 14 + seconds and the floating address answered from it after 23. +- **Nodes learn every control plane** from the leader (a new `ControlPlanes` + message, kept in `/var/lib/llmgw-agent/control-planes.json`), after the + ones in their configuration. +- **A node notices a dead control plane within a quarter of a minute.** A + leader whose machine stopped dead left its nodes' streams open until the + kernel gave up retransmitting, about fifteen minutes; the agent now pings + the stream after ten quiet seconds, and bounds dialling too. +- **`GET /leader`**: 200 on the control plane that leads, 503 on a standby, + open like `/ready` — what keepalived follows. +- **The store's access rules live in Patroni's shared configuration** for a + cluster set up by `server init --patroni`, so another control plane is let + in without touching each member. +- **The screens are windows**: full screen, a title bar, the steps or actions + down the left, the content in a bordered panel, the keys along the bottom, + questions and errors as dialogs over it — the setup, the console, adding a + node and one-machine init alike. Keyboard only. + ### Machines nothing may log into: `llmgw join` For sites that allow no ssh into a machine, the setup screens offer a fourth diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 7a345d2..d26030c 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21, 0} } // AgentMessage is anything a node tells the control plane. @@ -271,6 +271,7 @@ type ControlMessage struct { // *ControlMessage_FabricScan // *ControlMessage_FabricTest // *ControlMessage_Upgrade + // *ControlMessage_ControlPlanes Msg isControlMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -403,6 +404,15 @@ func (x *ControlMessage) GetUpgrade() *UpgradeAgent { return nil } +func (x *ControlMessage) GetControlPlanes() *ControlPlanes { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_ControlPlanes); ok { + return x.ControlPlanes + } + } + return nil +} + type isControlMessage_Msg interface { isControlMessage_Msg() } @@ -447,6 +457,10 @@ type ControlMessage_Upgrade struct { Upgrade *UpgradeAgent `protobuf:"bytes,10,opt,name=upgrade,proto3,oneof"` } +type ControlMessage_ControlPlanes struct { + ControlPlanes *ControlPlanes `protobuf:"bytes,11,opt,name=control_planes,json=controlPlanes,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -467,6 +481,56 @@ func (*ControlMessage_FabricTest) isControlMessage_Msg() {} func (*ControlMessage_Upgrade) isControlMessage_Msg() {} +func (*ControlMessage_ControlPlanes) isControlMessage_Msg() {} + +// ControlPlanes is every control plane an agent may dial, sent by the leader +// when an agent registers and whenever the list changes. The agent keeps it +// beside its configuration, so a second control plane added after the node +// was onboarded is one it fails over to without anybody editing the node. +type ControlPlanes struct { + state protoimpl.MessageState `protogen:"open.v1"` + Servers []string `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ControlPlanes) Reset() { + *x = ControlPlanes{} + mi := &file_llmgw_v1_agent_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ControlPlanes) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ControlPlanes) ProtoMessage() {} + +func (x *ControlPlanes) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ControlPlanes.ProtoReflect.Descriptor instead. +func (*ControlPlanes) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{2} +} + +func (x *ControlPlanes) GetServers() []string { + if x != nil { + return x.Servers + } + return nil +} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. @@ -557,7 +621,7 @@ type Register struct { func (x *Register) Reset() { *x = Register{} - mi := &file_llmgw_v1_agent_proto_msgTypes[2] + mi := &file_llmgw_v1_agent_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -569,7 +633,7 @@ func (x *Register) String() string { func (*Register) ProtoMessage() {} func (x *Register) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[2] + mi := &file_llmgw_v1_agent_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -582,7 +646,7 @@ func (x *Register) ProtoReflect() protoreflect.Message { // Deprecated: Use Register.ProtoReflect.Descriptor instead. func (*Register) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{2} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} } func (x *Register) GetNodeId() string { @@ -734,7 +798,7 @@ type Host struct { func (x *Host) Reset() { *x = Host{} - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -746,7 +810,7 @@ func (x *Host) String() string { func (*Host) ProtoMessage() {} func (x *Host) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -759,7 +823,7 @@ func (x *Host) ProtoReflect() protoreflect.Message { // Deprecated: Use Host.ProtoReflect.Descriptor instead. func (*Host) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} } func (x *Host) GetOsId() string { @@ -826,7 +890,7 @@ type Storage struct { func (x *Storage) Reset() { *x = Storage{} - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -838,7 +902,7 @@ func (x *Storage) String() string { func (*Storage) ProtoMessage() {} func (x *Storage) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -851,7 +915,7 @@ func (x *Storage) ProtoReflect() protoreflect.Message { // Deprecated: Use Storage.ProtoReflect.Descriptor instead. func (*Storage) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} } func (x *Storage) GetPath() string { @@ -918,7 +982,7 @@ type RunningShard struct { func (x *RunningShard) Reset() { *x = RunningShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -930,7 +994,7 @@ func (x *RunningShard) String() string { func (*RunningShard) ProtoMessage() {} func (x *RunningShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -943,7 +1007,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message { // Deprecated: Use RunningShard.ProtoReflect.Descriptor instead. func (*RunningShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} } func (x *RunningShard) GetReplicaId() string { @@ -1091,7 +1155,7 @@ type GPU struct { func (x *GPU) Reset() { *x = GPU{} - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1103,7 +1167,7 @@ func (x *GPU) String() string { func (*GPU) ProtoMessage() {} func (x *GPU) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1116,7 +1180,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message { // Deprecated: Use GPU.ProtoReflect.Descriptor instead. func (*GPU) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} } func (x *GPU) GetIndex() int32 { @@ -1280,7 +1344,7 @@ type FabricAttachment struct { func (x *FabricAttachment) Reset() { *x = FabricAttachment{} - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1292,7 +1356,7 @@ func (x *FabricAttachment) String() string { func (*FabricAttachment) ProtoMessage() {} func (x *FabricAttachment) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1305,7 +1369,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead. func (*FabricAttachment) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} } func (x *FabricAttachment) GetFabric() string { @@ -1347,7 +1411,7 @@ type LocalWeights struct { func (x *LocalWeights) Reset() { *x = LocalWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1359,7 +1423,7 @@ func (x *LocalWeights) String() string { func (*LocalWeights) ProtoMessage() {} func (x *LocalWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1372,7 +1436,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. func (*LocalWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *LocalWeights) GetPath() string { @@ -1428,7 +1492,7 @@ type NodeLoad struct { func (x *NodeLoad) Reset() { *x = NodeLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1440,7 +1504,7 @@ func (x *NodeLoad) String() string { func (*NodeLoad) ProtoMessage() {} func (x *NodeLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1453,7 +1517,7 @@ func (x *NodeLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeLoad.ProtoReflect.Descriptor instead. func (*NodeLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} } func (x *NodeLoad) GetCpuPercent() float64 { @@ -1586,7 +1650,7 @@ type GPUSensor struct { func (x *GPUSensor) Reset() { *x = GPUSensor{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1598,7 +1662,7 @@ func (x *GPUSensor) String() string { func (*GPUSensor) ProtoMessage() {} func (x *GPUSensor) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1611,7 +1675,7 @@ func (x *GPUSensor) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUSensor.ProtoReflect.Descriptor instead. func (*GPUSensor) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *GPUSensor) GetIndex() int32 { @@ -1825,7 +1889,7 @@ type ShardLoad struct { func (x *ShardLoad) Reset() { *x = ShardLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1837,7 +1901,7 @@ func (x *ShardLoad) String() string { func (*ShardLoad) ProtoMessage() {} func (x *ShardLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1850,7 +1914,7 @@ func (x *ShardLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead. func (*ShardLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *ShardLoad) GetShardId() string { @@ -1918,7 +1982,7 @@ type IBPort struct { func (x *IBPort) Reset() { *x = IBPort{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1930,7 +1994,7 @@ func (x *IBPort) String() string { func (*IBPort) ProtoMessage() {} func (x *IBPort) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1943,7 +2007,7 @@ func (x *IBPort) ProtoReflect() protoreflect.Message { // Deprecated: Use IBPort.ProtoReflect.Descriptor instead. func (*IBPort) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *IBPort) GetHca() string { @@ -2086,7 +2150,7 @@ type SubnetManager struct { func (x *SubnetManager) Reset() { *x = SubnetManager{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2098,7 +2162,7 @@ func (x *SubnetManager) String() string { func (*SubnetManager) ProtoMessage() {} func (x *SubnetManager) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2111,7 +2175,7 @@ func (x *SubnetManager) ProtoReflect() protoreflect.Message { // Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead. func (*SubnetManager) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *SubnetManager) GetGuid() string { @@ -2154,7 +2218,7 @@ type IMEX struct { func (x *IMEX) Reset() { *x = IMEX{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2166,7 +2230,7 @@ func (x *IMEX) String() string { func (*IMEX) ProtoMessage() {} func (x *IMEX) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2179,7 +2243,7 @@ func (x *IMEX) ProtoReflect() protoreflect.Message { // Deprecated: Use IMEX.ProtoReflect.Descriptor instead. func (*IMEX) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *IMEX) GetState() string { @@ -2211,7 +2275,7 @@ type GPUFabricState struct { func (x *GPUFabricState) Reset() { *x = GPUFabricState{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2223,7 +2287,7 @@ func (x *GPUFabricState) String() string { func (*GPUFabricState) ProtoMessage() {} func (x *GPUFabricState) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2236,7 +2300,7 @@ func (x *GPUFabricState) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead. func (*GPUFabricState) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *GPUFabricState) GetIndex() int32 { @@ -2351,7 +2415,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2363,7 +2427,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2376,7 +2440,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *Heartbeat) GetNodeId() string { @@ -2531,7 +2595,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2543,7 +2607,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2556,7 +2620,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *Provision) GetEngines() []*EngineWant { @@ -2580,7 +2644,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2592,7 +2656,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2605,7 +2669,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *EngineWant) GetEngine() string { @@ -2645,7 +2709,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2657,7 +2721,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2670,7 +2734,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *ProvisionStatus) GetEngine() string { @@ -2734,7 +2798,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2746,7 +2810,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2759,7 +2823,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *ShardStatus) GetNodeId() string { @@ -2833,7 +2897,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2845,7 +2909,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2858,7 +2922,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *FetchProgress) GetSource() string { @@ -2921,7 +2985,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2933,7 +2997,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2946,7 +3010,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *StartShard) GetReplicaId() string { @@ -3090,7 +3154,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3102,7 +3166,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3115,7 +3179,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *EngineProfile) GetName() string { @@ -3218,7 +3282,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3230,7 +3294,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3243,7 +3307,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *RemoveWeights) GetPaths() []string { @@ -3263,7 +3327,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3275,7 +3339,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3288,7 +3352,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} } func (x *StopShard) GetShardId() string { @@ -3308,7 +3372,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3320,7 +3384,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3333,7 +3397,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} } func (x *DrainShard) GetShardId() string { @@ -3355,7 +3419,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3367,7 +3431,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3380,7 +3444,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} } func (x *Ack) GetMessageId() string { @@ -3424,7 +3488,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3436,7 +3500,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3449,7 +3513,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} } func (x *LogsRequest) GetRequestId() string { @@ -3489,7 +3553,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3501,7 +3565,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3514,7 +3578,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} } func (x *LogsReply) GetRequestId() string { @@ -3560,7 +3624,7 @@ type FabricScanRequest struct { func (x *FabricScanRequest) Reset() { *x = FabricScanRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3572,7 +3636,7 @@ func (x *FabricScanRequest) String() string { func (*FabricScanRequest) ProtoMessage() {} func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3585,7 +3649,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead. func (*FabricScanRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} } func (x *FabricScanRequest) GetRequestId() string { @@ -3624,7 +3688,7 @@ type FabricScanReply struct { func (x *FabricScanReply) Reset() { *x = FabricScanReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3636,7 +3700,7 @@ func (x *FabricScanReply) String() string { func (*FabricScanReply) ProtoMessage() {} func (x *FabricScanReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3649,7 +3713,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead. func (*FabricScanReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} } func (x *FabricScanReply) GetRequestId() string { @@ -3705,7 +3769,7 @@ type FabricTestRequest struct { func (x *FabricTestRequest) Reset() { *x = FabricTestRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3717,7 +3781,7 @@ func (x *FabricTestRequest) String() string { func (*FabricTestRequest) ProtoMessage() {} func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3730,7 +3794,7 @@ func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead. func (*FabricTestRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} } func (x *FabricTestRequest) GetRequestId() string { @@ -3799,7 +3863,7 @@ type FabricTestReply struct { func (x *FabricTestReply) Reset() { *x = FabricTestReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3811,7 +3875,7 @@ func (x *FabricTestReply) String() string { func (*FabricTestReply) ProtoMessage() {} func (x *FabricTestReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3824,7 +3888,7 @@ func (x *FabricTestReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead. func (*FabricTestReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} } func (x *FabricTestReply) GetRequestId() string { @@ -3884,7 +3948,7 @@ type UpgradeAgent struct { func (x *UpgradeAgent) Reset() { *x = UpgradeAgent{} - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3896,7 +3960,7 @@ func (x *UpgradeAgent) String() string { func (*UpgradeAgent) ProtoMessage() {} func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3909,7 +3973,7 @@ func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead. func (*UpgradeAgent) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} } func (x *UpgradeAgent) GetRequestId() string { @@ -3946,7 +4010,7 @@ type UpgradeReply struct { func (x *UpgradeReply) Reset() { *x = UpgradeReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[35] + mi := &file_llmgw_v1_agent_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3958,7 +4022,7 @@ func (x *UpgradeReply) String() string { func (*UpgradeReply) ProtoMessage() {} func (x *UpgradeReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[35] + mi := &file_llmgw_v1_agent_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3971,7 +4035,7 @@ func (x *UpgradeReply) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead. func (*UpgradeReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} } func (x *UpgradeReply) GetRequestId() string { @@ -4007,7 +4071,7 @@ type LoraAdapter struct { func (x *LoraAdapter) Reset() { *x = LoraAdapter{} - mi := &file_llmgw_v1_agent_proto_msgTypes[36] + mi := &file_llmgw_v1_agent_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4019,7 +4083,7 @@ func (x *LoraAdapter) String() string { func (*LoraAdapter) ProtoMessage() {} func (x *LoraAdapter) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[36] + mi := &file_llmgw_v1_agent_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4032,7 +4096,7 @@ func (x *LoraAdapter) ProtoReflect() protoreflect.Message { // Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead. func (*LoraAdapter) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{37} } func (x *LoraAdapter) GetName() string { @@ -4064,7 +4128,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" + "fabricTest\x122\n" + "\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgradeB\x05\n" + - "\x03msg\"\x99\x04\n" + + "\x03msg\"\xdb\x04\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + @@ -4078,8 +4142,11 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" + "fabricTest\x122\n" + "\aupgrade\x18\n" + - " \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgradeB\x05\n" + - "\x03msg\"\x80\a\n" + + " \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgrade\x12@\n" + + "\x0econtrol_planes\x18\v \x01(\v2\x17.llmgw.v1.ControlPlanesH\x00R\rcontrolPlanesB\x05\n" + + "\x03msg\")\n" + + "\rControlPlanes\x12\x18\n" + + "\aservers\x18\x01 \x03(\tR\aservers\"\x80\a\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -4465,108 +4532,110 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 44) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage (*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage - (*Register)(nil), // 3: llmgw.v1.Register - (*Host)(nil), // 4: llmgw.v1.Host - (*Storage)(nil), // 5: llmgw.v1.Storage - (*RunningShard)(nil), // 6: llmgw.v1.RunningShard - (*GPU)(nil), // 7: llmgw.v1.GPU - (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights - (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad - (*GPUSensor)(nil), // 11: llmgw.v1.GPUSensor - (*ShardLoad)(nil), // 12: llmgw.v1.ShardLoad - (*IBPort)(nil), // 13: llmgw.v1.IBPort - (*SubnetManager)(nil), // 14: llmgw.v1.SubnetManager - (*IMEX)(nil), // 15: llmgw.v1.IMEX - (*GPUFabricState)(nil), // 16: llmgw.v1.GPUFabricState - (*Heartbeat)(nil), // 17: llmgw.v1.Heartbeat - (*Provision)(nil), // 18: llmgw.v1.Provision - (*EngineWant)(nil), // 19: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 20: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 21: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 22: llmgw.v1.FetchProgress - (*StartShard)(nil), // 23: llmgw.v1.StartShard - (*EngineProfile)(nil), // 24: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 25: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 26: llmgw.v1.StopShard - (*DrainShard)(nil), // 27: llmgw.v1.DrainShard - (*Ack)(nil), // 28: llmgw.v1.Ack - (*LogsRequest)(nil), // 29: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 30: llmgw.v1.LogsReply - (*FabricScanRequest)(nil), // 31: llmgw.v1.FabricScanRequest - (*FabricScanReply)(nil), // 32: llmgw.v1.FabricScanReply - (*FabricTestRequest)(nil), // 33: llmgw.v1.FabricTestRequest - (*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply - (*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent - (*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply - (*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter - nil, // 38: llmgw.v1.Register.LabelsEntry - nil, // 39: llmgw.v1.Register.EngineVersionsEntry - nil, // 40: llmgw.v1.GPU.NicPathsEntry - nil, // 41: llmgw.v1.IBPort.CountersEntry - nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 43: llmgw.v1.EngineProfile.EnvEntry + (*ControlPlanes)(nil), // 3: llmgw.v1.ControlPlanes + (*Register)(nil), // 4: llmgw.v1.Register + (*Host)(nil), // 5: llmgw.v1.Host + (*Storage)(nil), // 6: llmgw.v1.Storage + (*RunningShard)(nil), // 7: llmgw.v1.RunningShard + (*GPU)(nil), // 8: llmgw.v1.GPU + (*FabricAttachment)(nil), // 9: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 10: llmgw.v1.LocalWeights + (*NodeLoad)(nil), // 11: llmgw.v1.NodeLoad + (*GPUSensor)(nil), // 12: llmgw.v1.GPUSensor + (*ShardLoad)(nil), // 13: llmgw.v1.ShardLoad + (*IBPort)(nil), // 14: llmgw.v1.IBPort + (*SubnetManager)(nil), // 15: llmgw.v1.SubnetManager + (*IMEX)(nil), // 16: llmgw.v1.IMEX + (*GPUFabricState)(nil), // 17: llmgw.v1.GPUFabricState + (*Heartbeat)(nil), // 18: llmgw.v1.Heartbeat + (*Provision)(nil), // 19: llmgw.v1.Provision + (*EngineWant)(nil), // 20: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 21: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 22: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 23: llmgw.v1.FetchProgress + (*StartShard)(nil), // 24: llmgw.v1.StartShard + (*EngineProfile)(nil), // 25: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 26: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 27: llmgw.v1.StopShard + (*DrainShard)(nil), // 28: llmgw.v1.DrainShard + (*Ack)(nil), // 29: llmgw.v1.Ack + (*LogsRequest)(nil), // 30: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 31: llmgw.v1.LogsReply + (*FabricScanRequest)(nil), // 32: llmgw.v1.FabricScanRequest + (*FabricScanReply)(nil), // 33: llmgw.v1.FabricScanReply + (*FabricTestRequest)(nil), // 34: llmgw.v1.FabricTestRequest + (*FabricTestReply)(nil), // 35: llmgw.v1.FabricTestReply + (*UpgradeAgent)(nil), // 36: llmgw.v1.UpgradeAgent + (*UpgradeReply)(nil), // 37: llmgw.v1.UpgradeReply + (*LoraAdapter)(nil), // 38: llmgw.v1.LoraAdapter + nil, // 39: llmgw.v1.Register.LabelsEntry + nil, // 40: llmgw.v1.Register.EngineVersionsEntry + nil, // 41: llmgw.v1.GPU.NicPathsEntry + nil, // 42: llmgw.v1.IBPort.CountersEntry + nil, // 43: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 44: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ - 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 17, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 21, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 30, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 32, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply - 34, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply - 36, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply - 23, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 26, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 27, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 28, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 25, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 18, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 29, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest - 33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest - 35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent - 38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX - 0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 8, // 38: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 16, // 39: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState - 15, // 40: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX - 19, // 41: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter - 1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 48, // [48:49] is the sub-list for method output_type - 47, // [47:48] is the sub-list for method input_type - 47, // [47:47] is the sub-list for extension type_name - 47, // [47:47] is the sub-list for extension extendee - 0, // [0:47] is the sub-list for field type_name + 4, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register + 18, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 22, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 31, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 33, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply + 35, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply + 37, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply + 24, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 27, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 28, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 29, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 26, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 19, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 30, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 32, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 34, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest + 36, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent + 3, // 17: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes + 39, // 18: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 8, // 19: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 9, // 20: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 40, // 21: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 7, // 22: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 10, // 23: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 6, // 24: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 5, // 25: llmgw.v1.Register.host:type_name -> llmgw.v1.Host + 16, // 26: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX + 0, // 27: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 41, // 28: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 13, // 29: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 12, // 30: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor + 42, // 31: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 15, // 32: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 6, // 33: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 10, // 34: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 43, // 35: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 21, // 36: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 11, // 37: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 14, // 38: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 9, // 39: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 17, // 40: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState + 16, // 41: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX + 20, // 42: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 43: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 23, // 44: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 25, // 45: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 44, // 46: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 38, // 47: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter + 1, // 48: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 49: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 49, // [49:50] is the sub-list for method output_type + 48, // [48:49] is the sub-list for method input_type + 48, // [48:48] is the sub-list for extension type_name + 48, // [48:48] is the sub-list for extension extendee + 0, // [0:48] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -4594,6 +4663,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_FabricScan)(nil), (*ControlMessage_FabricTest)(nil), (*ControlMessage_Upgrade)(nil), + (*ControlMessage_ControlPlanes)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -4601,7 +4671,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 43, + NumMessages: 44, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 12e25cf..98caf13 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -43,9 +43,18 @@ message ControlMessage { FabricScanRequest fabric_scan = 8; FabricTestRequest fabric_test = 9; UpgradeAgent upgrade = 10; + ControlPlanes control_planes = 11; } } +// ControlPlanes is every control plane an agent may dial, sent by the leader +// when an agent registers and whenever the list changes. The agent keeps it +// beside its configuration, so a second control plane added after the node +// was onboarded is one it fails over to without anybody editing the node. +message ControlPlanes { + repeated string servers = 1; +} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. diff --git a/go.mod b/go.mod index 4a3c016..d41a6ea 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 go.opentelemetry.io/otel/sdk v1.46.0 go.opentelemetry.io/otel/trace v1.46.0 + go.yaml.in/yaml/v3 v3.0.5 golang.org/x/crypto v0.57.0 golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 @@ -74,7 +75,6 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect go.opentelemetry.io/otel/metric v1.46.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect - go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect diff --git a/go.sum b/go.sum index 626b7e0..037e610 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,8 @@ golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= diff --git a/internal/agent/agent.go b/internal/agent/agent.go index b224be5..2de863d 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -73,8 +73,11 @@ type Agent struct { lastEngineProbe time.Time probingEngines bool - mu sync.Mutex - inv discovery.Inventory + mu sync.Mutex + // serverList is every control plane to dial: the configured ones, and + // any the leader has named since. Read through servers(). + serverList []string + inv discovery.Inventory // shardReplica remembers which replica a shard belongs to, so a status // report can name both. The control plane keys on the replica. shardReplica map[string]string @@ -153,6 +156,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er load: discovery.NewLoadSampler(), sensors: discovery.NewSensorSampler(), wear: discovery.NewHealthSampler(), + serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()), } a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus} return a, nil @@ -185,7 +189,9 @@ func (a *Agent) Run(ctx context.Context) error { attempt := 0 next := 0 for ctx.Err() == nil { - server := a.cfg.Servers[next%len(a.cfg.Servers)] + // Read each time round: the leader may have named another since. + servers := a.servers() + server := servers[next%len(servers)] err := a.session(ctx, server) if ctx.Err() != nil { break @@ -200,12 +206,12 @@ func (a *Agent) Run(ctx context.Context) error { // No wait when there is somewhere else to try: the delay exists to // stop hammering one control plane, and the next address has not been // hammered. With a single address this is the old behaviour exactly. - if len(a.cfg.Servers) > 1 && next%len(a.cfg.Servers) != 0 { + if len(servers) > 1 && next%len(servers) != 0 { delay = 0 } a.log.Warn("disconnected from the control plane, retrying", "err", err, "server", server, - "next", a.cfg.Servers[next%len(a.cfg.Servers)], + "next", servers[next%len(servers)], "in", delay.Round(time.Millisecond), "attempt", attempt) select { @@ -792,6 +798,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha a.setProvision(m.Provision) case *llmgwv1.ControlMessage_Upgrade: go a.upgrade(ctx, m.Upgrade, out) + case *llmgwv1.ControlMessage_ControlPlanes: + a.setControlPlanes(m.ControlPlanes) case *llmgwv1.ControlMessage_Logs: reply := a.logsReply(m.Logs) // Off the receive loop: a full send queue must not stop commands being @@ -1084,7 +1092,9 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { if !cfg.Enabled() { return &http.Client{ Transport: &http2.Transport{ - AllowHTTP: true, + AllowHTTP: true, + ReadIdleTimeout: streamIdle, + PingTimeout: streamPing, DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr) }, @@ -1099,9 +1109,31 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { if err != nil { return nil, err } - return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil + return &http.Client{Transport: &http2.Transport{ + TLSClientConfig: tc, + ReadIdleTimeout: streamIdle, + PingTimeout: streamPing, + // Bounded, as the plaintext branch's is: dialling a control plane + // whose machine is off waits out the kernel's connect retries, + // about two minutes, before the next address is tried. + DialTLSContext: func(ctx context.Context, network, addr string, c *tls.Config) (net.Conn, error) { + d := tls.Dialer{NetDialer: &net.Dialer{Timeout: timeout}, Config: c} + return d.DialContext(ctx, network, addr) + }, + }}, nil } +// A stream to a control plane whose machine stopped dead — powered off, cut +// off — is never closed: nothing sends the FIN, and the agent's heartbeats sit +// in the socket's buffer until the kernel gives up retransmitting, fifteen +// minutes later. Found by switching off a leader: the standby took over in 14 +// seconds and the node stayed with the dead one. An HTTP/2 ping after ten +// quiet seconds, given five to answer, finds it in a quarter of a minute. +const ( + streamIdle = 10 * time.Second + streamPing = 5 * time.Second +) + // localizeSpec fills in what only this node knows. // // The interface NCCL bootstraps over is this node's, not the cluster's: the diff --git a/internal/agent/controlplanes.go b/internal/agent/controlplanes.go new file mode 100644 index 0000000..3a6afdb --- /dev/null +++ b/internal/agent/controlplanes.go @@ -0,0 +1,77 @@ +package agent + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" +) + +// controlPlanesFile keeps the control planes the leader last named, beside +// the agent's configuration rather than in it: the configuration is the +// operator's, and a list rewritten in it would be a file nobody wrote. +var controlPlanesFile = "/var/lib/llmgw-agent/control-planes.json" + +// servers is every address to dial, in the order to try them: the +// configuration's, then any the leader named that it lacks. +func (a *Agent) servers() []string { + a.mu.Lock() + defer a.mu.Unlock() + if len(a.serverList) == 0 { + return a.cfg.Servers + } + return a.serverList +} + +// mergeServers puts the leader's list after the configured one, without +// repeats, and says whether that changed anything. +func mergeServers(configured, told []string, secure bool) []string { + out := slices.Clone(configured) + for _, s := range told { + fixed, err := normalizeOne(s, secure) + if err != nil { + continue + } + if !slices.Contains(out, fixed) { + out = append(out, fixed) + } + } + return out +} + +// setControlPlanes takes the leader's list: used from the next reconnect, +// and kept for the next start. +func (a *Agent) setControlPlanes(m *llmgwv1.ControlPlanes) { + told := m.GetServers() + merged := mergeServers(a.cfg.Servers, told, a.cfg.TLS.Enabled()) + a.mu.Lock() + changed := !slices.Equal(merged, a.serverList) + a.serverList = merged + a.mu.Unlock() + if !changed { + return + } + a.log.Info("the control planes to dial", "servers", merged) + b, _ := json.Marshal(told) + if err := os.MkdirAll(filepath.Dir(controlPlanesFile), 0o755); err == nil { + tmp := controlPlanesFile + ".new" + if err := os.WriteFile(tmp, b, 0o644); err == nil { + _ = os.Rename(tmp, controlPlanesFile) + } + } +} + +// loadControlPlanes is the list kept from the last time the leader named it. +func loadControlPlanes(configured []string, secure bool) []string { + b, err := os.ReadFile(controlPlanesFile) + if err != nil { + return configured + } + var told []string + if json.Unmarshal(b, &told) != nil { + return configured + } + return mergeServers(configured, told, secure) +} diff --git a/internal/agent/controlplanes_test.go b/internal/agent/controlplanes_test.go new file mode 100644 index 0000000..3a68ab9 --- /dev/null +++ b/internal/agent/controlplanes_test.go @@ -0,0 +1,41 @@ +package agent + +import ( + "log/slog" + "path/filepath" + "slices" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +func TestAControlPlaneTheLeaderNamesIsDialledAndRemembered(t *testing.T) { + old := controlPlanesFile + controlPlanesFile = filepath.Join(t.TempDir(), "control-planes.json") + t.Cleanup(func() { controlPlanesFile = old }) + + configured := []string{"https://10.0.0.140:7443"} + // TLS on, as every onboarded node has it: an agent without it refuses + // https:// addresses, and the leader's are https://. + a := &Agent{cfg: Config{Servers: configured, TLS: tlsconf.Config{CA: "/etc/llmgw/ca.pem"}}, log: slog.New(slog.DiscardHandler)} + if got := a.servers(); !slices.Equal(got, configured) { + t.Fatalf("before anything is named: %v", got) + } + // The configured one stays first; the new one is added after it, once. + a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}}) + want := []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"} + if got := a.servers(); !slices.Equal(got, want) { + t.Errorf("after the leader named two: %v", got) + } + // And after a restart, before the leader says anything. + if got := loadControlPlanes(configured, true); !slices.Equal(got, want) { + t.Errorf("after a restart: %v", got) + } + // A list that drops the configured one does not drop it: the + // configuration is the operator's. + a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.150:7443"}}) + if got := a.servers(); got[0] != "https://10.0.0.140:7443" { + t.Errorf("the configured server was dropped: %v", got) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index e8586fe..4688a26 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -451,7 +451,7 @@ func newServerCmd() *cobra.Command { f.StringVar(&metricsListen, "metrics-listen", "", "serve metrics only on this address, safe to expose") f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)") - cmd.AddCommand(newServerInitCmd()) + cmd.AddCommand(newServerInitCmd(), newServerAddCmd()) cmd.AddCommand(newBackupCmd()) return cmd } @@ -513,6 +513,9 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live if h, ok := sink.(interface{ Store() *storepkg.DB }); ok { if st := h.Store(); st != nil { lead = control.NewLeadership(st, cfg.Control.LeaseTTL, log) + if r, ok := sink.(interface{ SetLeader(func() bool) }); ok { + r.SetLeader(lead.IsLeader) + } ctl.SetLeadership(lead) // The tenancy tree's capacity quotas: what each part is // guaranteed and may hold. @@ -542,6 +545,21 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live if lead != nil { svc.SetLeadership(lead) } + // The control planes agents may dial, from the store every control plane + // shares: told to each agent as it registers, and to all of them when + // `server add` records another. + if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil { + st := h.Store() + svc.SetControlPlanes(func(ctx context.Context) ([]string, error) { + cps, err := st.ControlPlanes(ctx) + urls := make([]string, len(cps)) + for i, c := range cps { + urls[i] = c.URL + } + return urls, err + }) + go svc.WatchControlPlanes(ctx, 30*time.Second) + } fwd.set(svc) // Every InfiniBand fabric mapped every five minutes, through one node. go svc.ScanFabrics(ctx, 5*time.Minute) diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go new file mode 100644 index 0000000..7dcbacf --- /dev/null +++ b/internal/cli/serveradd.go @@ -0,0 +1,459 @@ +package cli + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +// serverAdd makes another machine a control plane of this cluster: a standby +// that takes over when this one is lost. Run on a control plane that holds +// the authority, as root, like `node add`. +type serverAdd struct { + etcDir, pkiDir string + // hosts are more names to put in the new control plane's certificate. + hosts []string + // vip is a floating address, "10.0.0.160/24", to keep on whichever of the + // two leads; "" leaves client traffic to whatever is in front. + vip string + + remote onboard.Remote + // addr is the machine's address as given; ip what the others reach it at. + addr, ip, name string + out io.Writer +} + +func newServerAddCmd() *cobra.Command { + sa := serverAdd{etcDir: "/etc/llmgw", pkiDir: defaultPKIDir} + cmd := &cobra.Command{ + Use: "add [user@]address", + Short: "Make another machine a control plane of this cluster: a standby that takes over", + Long: "Run on a control plane, as root. It logs in to the machine with ssh, as\n" + + "node add does, and gives it this llmgw, a certificate from this\n" + + "cluster's authority, this control plane's configuration and the store's\n" + + "address, lets it into the store, and starts it. It stands by while this\n" + + "one leads, and takes over within seconds when this one is lost; every\n" + + "node is told of it and fails over to it without being touched.\n\n" + + "The store must be one both can reach — replicated with 'server init\n" + + "--patroni' — not a database on this machine.\n\n" + + "--vip keeps a floating address on whichever control plane leads, with\n" + + "keepalived on both: the address clients use for the gateway.\n\n" + + "Run it again at any time: it changes only what is missing.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if os.Geteuid() != 0 { + return errors.New("run it as root: it reads the certificate authority and the store's address") + } + user, host, ok := strings.Cut(args[0], "@") + if !ok { + user, host = "", args[0] + } + sa.remote = &onboard.SSH{Addr: host, User: user} + sa.addr = host + sa.out = cmd.OutOrStdout() + return sa.run(cmd.Context()) + }, + } + f := cmd.Flags() + f.StringArrayVar(&sa.hosts, "host", nil, "another name for the new control plane's certificate; repeatable") + f.StringVar(&sa.vip, "vip", "", "a floating address for whichever control plane leads, as 10.0.0.160/24") + f.StringVar(&sa.pkiDir, "pki", sa.pkiDir, "directory holding the certificate authority") + return cmd +} + +func (sa *serverAdd) done(format string, a ...any) { + fmt.Fprintf(sa.out, " ok "+format+"\n", a...) +} + +func (sa *serverAdd) kept(format string, a ...any) { + fmt.Fprintf(sa.out, " kept "+format+"\n", a...) +} + +func (sa *serverAdd) sh(ctx context.Context, script string, stdin []byte) (string, error) { + out, err := sa.remote.Run(ctx, script, stdin) + if err != nil { + return string(out), fmt.Errorf("on %s: %v: %s", sa.addr, err, strings.TrimSpace(string(out))) + } + return string(out), nil +} + +// put writes a file there, through standard input, with its mode and owner. +func (sa *serverAdd) put(ctx context.Context, path, mode, owner string, body []byte) error { + _, err := sa.sh(ctx, fmt.Sprintf(`set -e +sudo install -d -m 0755 %[1]s +sudo sh -c 'umask 077; cat > %[2]s.new' +sudo chmod %[3]s %[2]s.new +sudo chown %[4]s %[2]s.new +sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode, owner), body) + return err +} + +func (sa *serverAdd) run(ctx context.Context) error { + steps := []struct { + name string + fn func(context.Context) error + }{ + {"connect", sa.connect}, + {"binary", sa.binary}, + {"directories", sa.dirs}, + {"certificate", sa.certificate}, + {"configuration", sa.configuration}, + {"store", sa.storeAccess}, + {"service", sa.service}, + } + if sa.vip != "" { + steps = append(steps, struct { + name string + fn func(context.Context) error + }{"floating address", sa.floating}) + } + for _, s := range steps { + if err := s.fn(ctx); err != nil { + return fmt.Errorf("%s: %w", s.name, err) + } + } + fmt.Fprintf(sa.out, "\n%s is a control plane of this cluster, standing by while this one leads.\n", sa.name) + return nil +} + +func (sa *serverAdd) connect(ctx context.Context) error { + out, err := sa.sh(ctx, "hostname; sudo -n true && echo sudo-ok", nil) + if err != nil { + return fmt.Errorf("%w — it needs ssh and sudo without a password", err) + } + lines := strings.Split(strings.TrimSpace(out), "\n") + sa.name = strings.TrimSpace(lines[0]) + if sa.ip == "" { + if sa.ip, err = ipv4Of(sa.addr); err != nil { + return err + } + } + sa.done("%s (%s)", sa.name, sa.ip) + return nil +} + +// binary gives it this very llmgw, so the two control planes never differ. +func (sa *serverAdd) binary(ctx context.Context) error { + have, _ := sa.sh(ctx, "llmgw --version 2>/dev/null || true", nil) + if strings.TrimSpace(have) == "llmgw version "+version.Version { + sa.kept("llmgw %s", version.Version) + return nil + } + exe, err := os.Executable() + if err != nil { + return err + } + b, err := os.ReadFile(exe) + if err != nil { + return err + } + if _, err := sa.sh(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil { + return err + } + sa.done("llmgw %s, copied from this machine", version.Version) + return nil +} + +func (sa *serverAdd) dirs(ctx context.Context) error { + _, err := sa.sh(ctx, `set -e +id llmgw >/dev/null 2>&1 || sudo useradd --system --home /var/lib/llmgw --shell /usr/sbin/nologin llmgw +sudo install -d -m 0755 /etc/llmgw +sudo install -d -o llmgw -g llmgw -m 0700 /etc/llmgw/secrets +sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil) + if err == nil { + sa.done("user llmgw, /etc/llmgw and /var/lib/llmgw") + } + return err +} + +// certificate issues it a server certificate from this cluster's authority, +// for its name and addresses, unless it holds one that already names them. +func (sa *serverAdd) certificate(ctx context.Context) error { + ca, err := loadCA(sa.pkiDir) + if err != nil { + return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err) + } + ips, _ := sa.sh(ctx, "hostname -I", nil) + want := []string{sa.name, sa.ip} + for _, ip := range strings.Fields(ips) { + if net.ParseIP(ip).To4() != nil && !slices.Contains(want, ip) { + want = append(want, ip) + } + } + want = append(want, sa.hosts...) + + if have, _ := sa.sh(ctx, "sudo cat /etc/llmgw/server.pem 2>/dev/null || true", nil); have != "" { + if blk, _ := pem.Decode([]byte(have)); blk != nil { + if cert, err := x509.ParseCertificate(blk.Bytes); err == nil && cert.CheckSignatureFrom(ca.Cert) == nil { + names := strings.Split(sanNames(cert), ", ") + missing := false + for _, w := range want { + missing = missing || !slices.Contains(names, w) + } + if !missing { + sa.kept("certificate for %s", strings.Join(names, ", ")) + return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM) + } + } + } + } + leaf, err := ca.IssueServer(sa.name, want) + if err != nil { + return err + } + if err := sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM); err != nil { + return err + } + if err := sa.put(ctx, "/etc/llmgw/server.pem", "0644", "root:root", leaf.CertPEM); err != nil { + return err + } + if err := sa.put(ctx, "/etc/llmgw/server-key.pem", "0640", "root:llmgw", leaf.KeyPEM); err != nil { + return err + } + sa.done("certificate for %s, from this cluster's authority", strings.Join(want, ", ")) + return nil +} + +// configuration copies this control plane's, and the secrets beside it: the +// two must agree on everything, the store above all. +func (sa *serverAdd) configuration(ctx context.Context) error { + if have, _ := sa.sh(ctx, "sudo test -f /etc/llmgw/server.yaml && echo yes || true", nil); strings.TrimSpace(have) == "yes" { + sa.kept("/etc/llmgw/server.yaml (never overwritten)") + } else { + b, err := os.ReadFile(filepath.Join(sa.etcDir, "server.yaml")) + if err != nil { + return err + } + if err := sa.put(ctx, "/etc/llmgw/server.yaml", "0644", "root:root", b); err != nil { + return err + } + sa.done("/etc/llmgw/server.yaml, as this control plane's") + } + secrets, err := os.ReadDir(filepath.Join(sa.etcDir, "secrets")) + if err != nil && !os.IsNotExist(err) { + return err + } + for _, e := range secrets { + if e.IsDir() || strings.Contains(e.Name(), ".pre-") || strings.HasSuffix(e.Name(), ".new") { + continue + } + path := "/etc/llmgw/secrets/" + e.Name() + mine, err := os.ReadFile(filepath.Join(sa.etcDir, "secrets", e.Name())) + if err != nil { + return err + } + theirs, _ := sa.sh(ctx, "sudo cat "+path+" 2>/dev/null || true", nil) + switch { + case theirs == string(mine): + sa.kept("%s", path) + case theirs != "": + // Two control planes on two stores are two clusters. + return fmt.Errorf("%s there differs from this one's; remove it there if that machine is to join this cluster", path) + default: + if err := sa.put(ctx, path, "0600", "llmgw:llmgw", mine); err != nil { + return err + } + sa.done("%s", path) + } + } + return nil +} + +// storeAccess lets it into the store, and records both control planes for +// the nodes to be told. +func (sa *serverAdd) storeAccess(ctx context.Context) error { + cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml")) + if err != nil { + return err + } + addr, err := store.ResolveAddress(cfg.Auth.Store) + if err != nil { + return err + } + addr = strings.TrimSpace(addr) + if !store.IsPostgres(addr) { + return errors.New("the store is SQLite on this machine; a second control plane needs one both reach — run 'llmgw server init --patroni' first") + } + hosts, _, _, err := store.SplitPostgres(addr) + if err != nil { + return err + } + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil && ip.IsLoopback() || h == "localhost" { + return fmt.Errorf("the store is at %s, on this machine only; replicate it first with 'llmgw server init --patroni'", h) + } + } + switch { + case slices.Contains(hosts, sa.ip): + sa.kept("store: %s is one of its members", sa.ip) + case len(hosts) > 1: + changed, err := pgha.AllowClient(ctx, hosts, sa.ip) + if err != nil { + return err + } + if changed { + sa.done("store: %s let in, through Patroni", sa.ip) + } else { + sa.kept("store: %s already let in", sa.ip) + } + default: + sa.kept("store at %s: make sure it lets %s in", hosts[0], sa.ip) + } + + st, err := store.Open(ctx, addr) + if err != nil { + return err + } + defer st.Close() + _, port, _ := net.SplitHostPort(cfg.AgentListen) + if port == "" { + port = "7443" + } + self, err := sourceIPToward(sa.ip) + if err != nil { + return err + } + me, _ := os.Hostname() + for _, cp := range []store.ControlPlane{ + {URL: "https://" + net.JoinHostPort(self, port), Name: me}, + {URL: "https://" + net.JoinHostPort(sa.ip, port), Name: sa.name, AddedAt: time.Now().Add(time.Second)}, + } { + if err := st.PutControlPlane(ctx, cp); err != nil { + return err + } + } + sa.done("nodes will be told of both control planes within half a minute") + return nil +} + +// service installs the unit and starts it, then waits for it to answer. +func (sa *serverAdd) service(ctx context.Context) error { + if err := sa.put(ctx, "/etc/systemd/system/llmgw-server.service", "0644", "root:root", []byte(serverUnit)); err != nil { + return err + } + if _, err := sa.sh(ctx, "sudo systemctl daemon-reload && sudo systemctl enable llmgw-server >/dev/null 2>&1 && sudo systemctl restart llmgw-server", nil); err != nil { + return err + } + deadline := time.Now().Add(90 * time.Second) + for { + code, _ := sa.sh(ctx, "curl -s -o /dev/null -m 2 -w '%{http_code}' http://127.0.0.1:8080/leader || true", nil) + switch strings.TrimSpace(code) { + case "200": + sa.done("llmgw-server running, and leading") + return nil + case "503": + sa.done("llmgw-server running, standing by") + return nil + } + if time.Now().After(deadline) { + logs, _ := sa.sh(ctx, "sudo journalctl -u llmgw-server -n 15 --no-pager", nil) + return fmt.Errorf("it did not answer within 90s:\n%s", logs) + } + time.Sleep(2 * time.Second) + } +} + +// floating keeps an address on whichever of the two leads, with keepalived +// on both, following /leader. +func (sa *serverAdd) floating(ctx context.Context) error { + vip, nw, err := net.ParseCIDR(sa.vip) + if err != nil || vip.To4() == nil { + return fmt.Errorf("--vip %q is not an IPv4 address with its prefix, like 10.0.0.160/24", sa.vip) + } + prefix, _ := nw.Mask.Size() + pass, err := vrrpPassword(sa.etcDir) + if err != nil { + return err + } + here := pgha.Local{} + for _, m := range []struct { + r onboard.Remote + name string + priority int + }{{here, "this machine", 110}, {sa.remote, sa.name, 100}} { + out, err := m.r.Run(ctx, fmt.Sprintf(`set -e +command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; } +ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil) + if err != nil { + return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out))) + } + iface := strings.TrimSpace(string(out)) + conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass) + if out, err := m.r.Run(ctx, `set -e +sudo install -d -m 0755 /etc/keepalived +sudo sh -c 'umask 077; cat > /etc/keepalived/keepalived.conf' +sudo systemctl enable keepalived >/dev/null 2>&1 +sudo systemctl restart keepalived`, []byte(conf)); err != nil { + return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out))) + } + } + sa.done("%s follows whichever control plane leads (keepalived on both)", sa.vip) + return nil +} + +// vrrpPassword is the shared secret of the two keepalived, made once and +// kept with the other secrets, so running this again does not split them. +func vrrpPassword(etcDir string) (string, error) { + p := filepath.Join(etcDir, "secrets", "vrrp") + if b, err := os.ReadFile(p); err == nil && len(bytes.TrimSpace(b)) > 0 { + return string(bytes.TrimSpace(b)), nil + } + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + return "", err + } + pass := hex.EncodeToString(b) // keepalived reads eight characters + if err := os.WriteFile(p, []byte(pass+"\n"), 0o600); err != nil { + return "", err + } + return pass, nil +} + +func keepalivedConf(iface, vip string, prefix, priority int, pass string) string { + return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads. +vrrp_script llmgw_leader { + script "/usr/bin/curl -fsS -o /dev/null -m 2 http://127.0.0.1:8080/leader" + interval 2 + fall 2 + rise 2 +} + +vrrp_instance llmgw { + state BACKUP + interface %s + virtual_router_id 51 + priority %d + advert_int 1 + authentication { + auth_type PASS + auth_pass %s + } + virtual_ipaddress { + %s/%d + } + track_script { + llmgw_leader + } +} +`, iface, priority, pass, vip, prefix) +} diff --git a/internal/cli/tui_frame.go b/internal/cli/tui_frame.go new file mode 100644 index 0000000..1600a56 --- /dev/null +++ b/internal/cli/tui_frame.go @@ -0,0 +1,184 @@ +package cli + +import ( + "strings" + + "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" +) + +// The frame every full-screen llmgw screen is drawn in: a title bar, a +// sidebar of steps or actions on the left, the content in a bordered panel on +// the right, the keys along the bottom — and a dialog over it all when one is +// open. One frame, so the setup and the console read as one program. +// +// Keyboard only: these run on serial consoles and IPMI as often as in a +// desktop terminal. + +// navItem is one line of the sidebar. +type navItem struct { + Label string + // State is "done", "current", "todo" for steps; "" for menu entries, + // where Current marks the one focused. + State string + Current bool + // Dim greys an entry that cannot be chosen now. + Dim bool +} + +type frame struct { + Width, Height int + Title string // left of the title bar + Where string // right of the title bar: the machine + NavTitle string + Nav []navItem + PanelTitle string + Body string + // Scroll is how many of the body's lines are scrolled off the top. + Scroll int + Keys string + // Dialog, when set, is drawn over the middle of the screen. + Dialog string + DialogTitle string +} + +var ( + fBar = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.AdaptiveColor{Light: "#FFFFFF", Dark: "#0B1B1B"}).Background(tuiAccent) + fBorder = lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(tuiDim) + fFocus = lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(tuiAccent) + fKeys = lipgloss.NewStyle().Foreground(tuiDim) + fSelected = lipgloss.NewStyle().Bold(true).Foreground(tuiAccent) + fDialog = lipgloss.NewStyle().Border(lipgloss.DoubleBorder()).BorderForeground(tuiAccent).Padding(1, 2) +) + +const navWidth = 24 + +// render draws the whole screen, exactly Width by Height. +func (f frame) render() string { + w, h := max(f.Width, 60), max(f.Height, 16) + + title := " " + f.Title + right := f.Where + " " + gap := max(1, w-ansi.StringWidth(title)-ansi.StringWidth(right)) + bar := fBar.Render(title + strings.Repeat(" ", gap) + right) + + // Panels fill what the bars leave: one line each above and below. + ph := h - 2 + inner := ph - 2 // borders + + var nav strings.Builder + for _, it := range f.Nav { + mark := " " + label := it.Label + switch { + case it.State == "done": + mark = sOK.Render("✓ ") + case it.State == "current": + mark = fSelected.Render("› ") + label = fSelected.Render(label) + case it.Current: + mark = fSelected.Render("› ") + label = fSelected.Render(label) + case it.Dim || it.State == "todo": + label = sDim.Render(label) + } + nav.WriteString(truncateWidth(mark+label, navWidth-2) + "\n") + } + left := box(fBorder, f.NavTitle, nav.String(), navWidth, inner) + + pw := w - navWidth - 2 - 2 + lines := strings.Split(strings.TrimRight(f.Body, "\n"), "\n") + scroll := min(max(f.Scroll, 0), max(0, len(lines)-inner)) + visible := lines[scroll:] + if len(visible) > inner { + visible = visible[:inner] + // Said in the last line, where it is seen. + visible[inner-1] = sDim.Render(" ↓ more — PgDn") + } + for i := range visible { + visible[i] = truncateWidth(visible[i], pw) + } + panelTitle := f.PanelTitle + if scroll > 0 { + panelTitle += " ↑" + } + main := box(fFocus, panelTitle, strings.Join(visible, "\n"), pw, inner) + + keys := fKeys.Render(" " + truncateWidth(f.Keys, w-2)) + screen := bar + "\n" + lipgloss.JoinHorizontal(lipgloss.Top, left, main) + "\n" + keys + + if f.Dialog != "" { + dw := min(w-8, max(50, maxLineWidth(f.Dialog)+6)) + d := fDialog.Width(dw).Render(f.Dialog) + if f.DialogTitle != "" { + d = titled(d, f.DialogTitle) + } + screen = overlay(screen, d, w, h) + } + return screen +} + +// box draws content in a border of the given inner size, with a title set +// into the top edge. +func box(style lipgloss.Style, title, content string, width, height int) string { + b := style.Width(width).Height(height).Render(content) + if title == "" { + return b + } + return titled(b, title) +} + +// titled sets " title " into a box's top border, after its corner. +func titled(b, title string) string { + lines := strings.Split(b, "\n") + if len(lines) == 0 { + return b + } + top := lines[0] + t := " " + title + " " + tw := ansi.StringWidth(t) + if ansi.StringWidth(top) < tw+4 { + return b + } + lines[0] = ansi.Truncate(top, 2, "") + sTitle.Render(t) + ansi.TruncateLeft(top, 2+tw, "") + return strings.Join(lines, "\n") +} + +// overlay draws top over the middle of base, both w by h. +func overlay(base, top string, w, h int) string { + bl := strings.Split(base, "\n") + tl := strings.Split(top, "\n") + tw := maxLineWidth(top) + x := max(0, (w-tw)/2) + y := max(0, (h-len(tl))/2) + for i, l := range tl { + row := y + i + if row >= len(bl) { + break + } + under := bl[row] + left := ansi.Truncate(under, x, "") + if pad := x - ansi.StringWidth(left); pad > 0 { + left += strings.Repeat(" ", pad) + } + right := ansi.TruncateLeft(under, x+ansi.StringWidth(l), "") + bl[row] = left + l + right + } + return strings.Join(bl, "\n") +} + +func maxLineWidth(s string) int { + n := 0 + for _, l := range strings.Split(s, "\n") { + n = max(n, ansi.StringWidth(l)) + } + return n +} + +// truncateWidth cuts a line, colours and all, to w cells. +func truncateWidth(s string, w int) string { + if ansi.StringWidth(s) <= w { + return s + } + return ansi.Truncate(s, max(w-1, 0), "…") +} diff --git a/internal/cli/tui_home.go b/internal/cli/tui_home.go index ff329a1..013f536 100644 --- a/internal/cli/tui_home.go +++ b/internal/cli/tui_home.go @@ -13,7 +13,6 @@ import ( "time" tea "github.com/charmbracelet/bubbletea" - "github.com/charmbracelet/lipgloss" "github.com/mattn/go-isatty" "github.com/spf13/cobra" @@ -88,6 +87,7 @@ type consoleModel struct { prompt textField action consoleAction width int + height int spin spinnerModel loaded bool } @@ -122,7 +122,7 @@ func (m consoleModel) Init() tea.Cmd { return tea.Batch(m.fetch, m.spin.Tick) } func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: - m.width = msg.Width + m.width, m.height = msg.Width, msg.Height case spinTickMsg: var cmd tea.Cmd m.spin, cmd = m.spin.Update(msg) @@ -183,28 +183,30 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } func (m consoleModel) View() string { - var b strings.Builder host, _ := os.Hostname() - b.WriteString(sTitle.Render("llmgw · console") + " " + sDim.Render(host) + "\n\n") + var nav []navItem + for i, it := range m.items() { + nav = append(nav, navItem{Label: it.label, Current: i == m.focus}) + } + var body string switch { case !m.loaded: - b.WriteString(" " + m.spin.View() + " asking the control plane…\n") + body = "\n " + m.spin.View() + " asking the control plane…" case m.err != nil: - b.WriteString(sBad.Render("The control plane does not answer: ") + m.err.Error() + "\n" + - sDim.Render("systemctl status llmgw-server says why.") + "\n") + body = "\n " + sBad.Render("The control plane does not answer:") + "\n " + m.err.Error() + + "\n\n " + sDim.Render("systemctl status llmgw-server says why.") default: - b.WriteString(m.summary) + body = "\n" + indent(m.summary, " ") } - b.WriteString("\n" + lipgloss.NewStyle().Bold(true).Render("Do") + "\n") - for i, it := range m.items() { - line := " " + it.label - if i == m.focus { - line = sTitle.Render("› " + it.label) - } - b.WriteString(line + "\n") - if i == m.focus && it.command != "" { - b.WriteString(" " + sDim.Render(it.command) + "\n") - } + if it := m.items()[m.focus]; it.command != "" { + body += "\n\n " + sDim.Render("On the command line:") + "\n " + it.command + } + f := frame{ + Width: m.width, Height: m.height, + Title: "llmgw console", Where: host, + NavTitle: "Do", Nav: nav, + PanelTitle: "Cluster", Body: body, + Keys: "↑↓ choose · Enter do it · Q leave · refreshed every five seconds", } if m.asking { it := m.items()[m.focus] @@ -212,11 +214,18 @@ func (m consoleModel) View() string { if it.n > 1 { ask = fmt.Sprintf("%s %d of %d", ask, len(m.asked)+1, it.n) } - b.WriteString("\n" + ask + "\n " + m.prompt.View() + "\n" + sDim.Render("enter · esc to cancel") + "\n") - } else { - b.WriteString("\n" + sDim.Render("↑↓ to choose · enter · q to leave · refreshed every five seconds") + "\n") + f.DialogTitle = it.label + f.Dialog = ask + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel") } - return b.String() + return f.render() +} + +func indent(s, pre string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i := range lines { + lines[i] = pre + lines[i] + } + return strings.Join(lines, "\n") } // runConsole shows the console, runs what is chosen on it, and comes back. @@ -229,8 +238,8 @@ func runConsole(ctx context.Context, out io.Writer) error { } } for { - m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100} - final, err := tea.NewProgram(m, tea.WithOutput(out)).Run() + m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30} + final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run() if err != nil { return err } diff --git a/internal/cli/tui_init.go b/internal/cli/tui_init.go index d1a3fdc..5a9f23f 100644 --- a/internal/cli/tui_init.go +++ b/internal/cli/tui_init.go @@ -77,10 +77,14 @@ type initModel struct { next []string // the closing "Next:" text spin spinnerModel width int - done bool - err error - quit bool - start func() + height int + // framed draws it in the full-screen frame; off when another screen + // shows it in its own panel. + framed bool + done bool + err error + quit bool + start func() } func newInitModel(in *serverInit, found []string, askForm bool) initModel { @@ -118,7 +122,10 @@ type startInitMsg struct{} func (m initModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: - m.width = msg.Width + m.width, m.height = msg.Width, msg.Height + if m.framed { + m.width = msg.Width - navWidth - 6 + } case spinTickMsg: var cmd tea.Cmd m.spin, cmd = m.spin.Update(msg) @@ -132,12 +139,18 @@ func (m initModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.addLine(string(msg)) case initDoneMsg: m.done, m.err = true, msg.err + if m.framed { + return m, nil // left on screen until a key: it is gone once closed + } return m, tea.Quit case tea.KeyMsg: if msg.Type == tea.KeyCtrlC { m.quit = true return m, tea.Quit } + if m.done && m.framed && (msg.Type == tea.KeyEnter || msg.String() == "q") { + return m, tea.Quit + } if m.form { return m.formKey(msg) } @@ -240,6 +253,30 @@ func (m *initModel) moveFocus(d int) { } func (m initModel) View() string { + if !m.framed { + return m.plainView() + } + body := m.plainView() + if i := strings.Index(body, "\n\n"); i >= 0 { + body = body[i+1:] + } + steps := []navItem{{Label: "Start", State: "done"}, {Label: "Settings", State: "current"}, {Label: "Install", State: "todo"}} + keys := "Tab/↑↓ move · Space tick · Enter on Set up · Ctrl+C leave" + if !m.form { + steps[1].State, steps[2].State = "done", "current" + keys = "Ctrl+C stop" + if m.done { + steps[2].State = "done" + keys = "Enter close" + } + } + host, _ := os.Hostname() + return frame{Width: m.width + navWidth + 6, Height: m.height, Title: "llmgw setup", Where: host, + NavTitle: "Steps", Nav: steps, PanelTitle: "This machine as a control plane", Body: body, Keys: keys}.render() +} + +// plainView is the screen as text, for a panel or a terminal that is not one. +func (m initModel) plainView() string { var b strings.Builder b.WriteString(sTitle.Render("llmgw · setting up this machine as a control plane") + "\n\n") if m.form { @@ -338,7 +375,9 @@ func (m initModel) formView() string { if m.formError != "" { b.WriteString("\n" + sBad.Render(m.formError) + "\n") } - b.WriteString("\n" + sDim.Render("tab or ↑↓ to move · space to tick · enter on Set up to start · ctrl+c to leave") + "\n") + if !m.framed { // the frame's key bar says it + b.WriteString("\n" + sDim.Render("tab or ↑↓ to move · space to tick · enter on Set up to start · ctrl+c to leave") + "\n") + } return b.String() } @@ -350,6 +389,7 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error { found = append(found, ip.String()) } m := newInitModel(in, found, in.admin == "") + m.framed = true var p *tea.Program var ( @@ -373,7 +413,7 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error { p.Send(initDoneMsg{err}) }() } - p = tea.NewProgram(m, tea.WithOutput(out)) + p = tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()) final, err := p.Run() if err != nil { return err diff --git a/internal/cli/tui_node.go b/internal/cli/tui_node.go index 7f655e2..142d2f8 100644 --- a/internal/cli/tui_node.go +++ b/internal/cli/tui_node.go @@ -68,6 +68,7 @@ type nodeModel struct { spin spinnerModel cancel context.CancelFunc width int + height int done bool err error node string @@ -91,7 +92,7 @@ func (m nodeModel) Init() tea.Cmd { return m.spin.Tick } func (m nodeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: - m.width = msg.Width + m.width, m.height = msg.Width, msg.Height case spinTickMsg: var cmd tea.Cmd m.spin, cmd = m.spin.Update(msg) @@ -131,9 +132,13 @@ func (m nodeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case askContinueMsg: m.confirm = &msg case finishMsg: + // Left on the screen until a key: the frame is gone once it closes. m.done, m.err, m.node, m.summary = true, msg.err, msg.node, msg.summary - return m, tea.Quit + return m, nil case tea.KeyMsg: + if m.done && (msg.Type == tea.KeyEnter || msg.String() == "q" || msg.Type == tea.KeyEsc) { + return m, tea.Quit + } return m.key(msg) } return m, nil @@ -197,10 +202,52 @@ func (m *nodeModel) move(d int) { m.inputs[m.focus].Focus() } +// View draws the onboarding in the frame: its steps down the side, what each +// did in the panel, a question as a dialog. func (m nodeModel) View() string { + var nav []navItem + current := false + for _, step := range onboard.Steps { + st, ok := m.steps[step] + state := "todo" + switch { + case ok && st.level == onboard.Running && !m.done && !current: + state, current = "current", true + case ok && st.level != onboard.Running: + state = "done" + } + nav = append(nav, navItem{Label: step, State: state}) + } + keys := "Ctrl+C stop" + if m.done { + keys = "Enter close" + } + host, _ := os.Hostname() + f := frame{ + Width: m.width, Height: m.height, Title: "llmgw · add a node", Where: host, + NavTitle: "Steps", Nav: nav, PanelTitle: m.title, Body: "\n" + m.content(), Keys: keys, + } + switch { + case m.fabrics != nil: + var q strings.Builder + q.WriteString("These adapters are up. Which fabric is each group on?\n") + q.WriteString(sDim.Render("Give each the name it has on every other node attached to it.\nLeave a name empty to leave that group out.") + "\n\n") + for i, fb := range m.fabrics.suggested { + fmt.Fprintf(&q, " %s %s\n", m.inputs[i].View(), sDim.Render(strings.Join(fb.HCAs, ", "))) + } + q.WriteString("\n" + sDim.Render("Tab next · Enter use these · Esc none")) + f.DialogTitle, f.Dialog = "Fabrics", q.String() + case m.confirm != nil: + f.DialogTitle = "Problems found" + f.Dialog = fmt.Sprintf("doctor found %d problem(s) on the machine.\nAdd the node anyway?\n\n%s", m.confirm.fails, sDim.Render("Y yes · N no")) + } + return f.render() +} + +// content is the onboarding's steps and notes, as the panel shows them. +func (m nodeModel) content() string { var b strings.Builder - b.WriteString(sTitle.Render(m.title) + "\n\n") - detailWidth := max(20, m.width-22) + detailWidth := max(20, m.width-navWidth-26) for _, step := range onboard.Steps { st, ok := m.steps[step] var mark string @@ -238,26 +285,12 @@ func (m nodeModel) View() string { } switch { - case m.fabrics != nil: - var q strings.Builder - q.WriteString("These adapters are up. Which fabric is each group on?\n") - q.WriteString(sDim.Render("Give each the name it has on every other node attached to it.\nLeave a name empty to leave that group out.") + "\n\n") - for i, f := range m.fabrics.suggested { - fmt.Fprintf(&q, " %s %s\n", m.inputs[i].View(), sDim.Render(strings.Join(f.HCAs, ", "))) - } - q.WriteString("\n" + sDim.Render("tab next · enter use these · esc none")) - b.WriteString("\n" + sBox.Render(q.String()) + "\n") - case m.confirm != nil: - q := fmt.Sprintf("doctor found %d problem(s) above.\nAdd the node anyway? %s", m.confirm.fails, sDim.Render("y / N")) - b.WriteString("\n" + sBox.BorderForeground(tuiWarn).Render(q) + "\n") case m.done && m.err != nil: b.WriteString("\n" + sBad.Render("✗ ") + wrap(m.err.Error(), m.width-4, " ") + "\n") case m.done: b.WriteString("\n" + sOK.Render("✓ ") + m.summary + "\n") case m.quitAsked: b.WriteString("\n" + sDim.Render("stopped") + "\n") - default: - b.WriteString("\n" + sDim.Render("ctrl+c to stop") + "\n") } return b.String() } @@ -298,7 +331,7 @@ func (na *nodeAdd) runTUI(ctx context.Context, o onboard.Options, title string) ctx, cancel := context.WithCancel(ctx) defer cancel() m := newNodeModel(title, cancel) - p := tea.NewProgram(m) + p := tea.NewProgram(m, tea.WithAltScreen()) emit := func(e onboard.Event) { p.Send(evMsg(e)) } o.AskFabrics = na.rememberFabrics(func(s []onboard.Fabric) []onboard.Fabric { @@ -343,6 +376,8 @@ func (na *nodeAdd) runTUI(ctx context.Context, o onboard.Options, title string) return "", err } fm := final.(nodeModel) + // The screen is gone; what it ended with is said again where it stays. + fmt.Println(fm.content()) if fm.quitAsked && !fm.done { cancel() return fm.node, fmt.Errorf("stopped; the node may be half set up, and running this again finishes it") diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go index 9ed5613..f4a1cdf 100644 --- a/internal/cli/tui_setup.go +++ b/internal/cli/tui_setup.go @@ -169,7 +169,10 @@ type setupModel struct { spin spinnerModel formErr string width int - quit bool + height int + // scroll is how far the panel is scrolled, reset with each page. + scroll int + quit bool // single is the choice of one machine: the screen closes and init's own // form takes over. single bool @@ -182,7 +185,7 @@ func newSetupModel(host string, ips []string) setupModel { names.Placeholder, names.CharLimit, names.Width = "llm.example.com, 203.0.113.10", 400, 40 return setupModel{ ref: &progRef{}, host: host, ips: ips, admin: admin, names: names, - db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, + db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30, log: &[]string{}, logMu: &sync.Mutex{}, } } @@ -199,8 +202,8 @@ func (m setupModel) thisAddr() string { func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch msg := msg.(type) { case tea.WindowSizeMsg: - m.width = msg.Width - m.run.width = msg.Width + m.width, m.height = msg.Width, msg.Height + m.run.width = msg.Width - navWidth - 6 case spinTickMsg: var cmd tea.Cmd m.spin, cmd = m.spin.Update(msg) @@ -224,12 +227,32 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.quit = true return m, tea.Quit } - return m.key(msg) + before := m.page + next, cmd := m.key(msg) + if nm := next.(setupModel); nm.page != before { + // A new page starts at its top. + nm.scroll = 0 + return nm, cmd + } + return next, cmd } return m, nil } func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { + // A dialog open takes the next key, whatever it is, to close it. + if m.formErr != "" { + m.formErr = "" + return m, nil + } + switch k.Type { + case tea.KeyPgDown: + m.scroll += max(1, m.height-8) + return m, nil + case tea.KeyPgUp: + m.scroll = max(0, m.scroll-max(1, m.height-8)) + return m, nil + } if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking { m.page-- if m.page == pWait { @@ -627,78 +650,95 @@ func (m *setupModel) startRun() tea.Cmd { } } -func (m setupModel) View() string { - var b strings.Builder - b.WriteString(sTitle.Render("llmgw · setup") + " " + sDim.Render(m.crumbs()) + "\n\n") - switch m.page { - case pWelcome: - b.WriteString(m.welcomeView()) - case pAdmin: - b.WriteString(m.adminView()) - case pMachines: - b.WriteString(m.machinesView()) - case pWait: - b.WriteString(m.waitView()) - case pCheck: - b.WriteString(m.checkView()) - case pPlan: - b.WriteString(m.planView()) - case pRun: - v := m.run.View() - if i := strings.Index(v, "\n\n"); i >= 0 { - v = v[i+2:] // its own title; this screen has one - } - b.WriteString(v) - if m.finished { - if m.run.err == nil { - b.WriteString("\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n") - } - b.WriteString("\n" + sDim.Render("enter or q to leave") + "\n") - } - } - if m.formErr != "" { - b.WriteString("\n" + sBad.Render(m.formErr) + "\n") - } - return b.String() +// setupSteps are the sidebar's steps, by page. +var setupSteps = []struct { + label string + page setupPage +}{ + {"Start", pWelcome}, {"Administrator", pAdmin}, {"Machines", pMachines}, + {"Checks", pCheck}, {"Plan", pPlan}, {"Install", pRun}, } -func (m setupModel) crumbs() string { - steps := []string{"start", "administrator", "machines", "checks", "plan", "install"} - if m.choice == 0 { - return steps[0] +func (m setupModel) View() string { + at := m.page + if at == pWait { + at = pMachines } - at := int(m.page) - switch { - case m.page == pWait: - at = int(pMachines) - case m.page > pWait: - at-- - } - var out []string - for i, s := range steps { - if i == at { - s = lipgloss.NewStyle().Bold(true).Render(s) + var nav []navItem + for _, st := range setupSteps { + if m.choice == 0 && st.page > pWelcome { + break } - out = append(out, s) + state := "todo" + switch { + case st.page < at || (st.page == pRun && m.finished && m.run.err == nil): + state = "done" + case st.page == at: + state = "current" + } + nav = append(nav, navItem{Label: st.label, State: state}) } - return strings.Join(out, " › ") + title, body, keys := m.pageView() + f := frame{ + Width: m.width, Height: m.height, + Title: "llmgw setup", Where: m.host + " · " + m.thisAddr(), + NavTitle: "Steps", Nav: nav, + PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys, + } + if m.formErr != "" { + f.DialogTitle, f.Dialog = "Not yet", m.formErr+"\n\n"+sDim.Render("any key to go back") + } + return f.render() +} + +// pageView is the panel for the page: its title, what is in it, and its keys. +func (m setupModel) pageView() (string, string, string) { + switch m.page { + case pWelcome: + return "Start", m.welcomeView(), "↑↓ choose · Enter go on · Ctrl+C leave" + case pAdmin: + return "Administrator", m.adminView(), "Tab/↑↓ move · Enter next · Esc back · Ctrl+C leave" + case pMachines: + return "Database machines", m.machinesView(), "Tab/↑↓ move · ←→ how to log in · Enter next · Esc back · PgUp/PgDn scroll" + case pWait: + return "Waiting for machines to join", m.waitView(), "Esc back · PgUp/PgDn scroll · Ctrl+C leave" + case pCheck: + keys := "R check again · Esc back · PgUp/PgDn scroll" + if !m.checking && m.report.Worst() < preflight.Fail { + keys = "Enter see the plan · " + keys + } + return "Checks", m.checkView(), keys + case pPlan: + return "Plan", m.planView(), "Enter install · Esc back · PgUp/PgDn scroll · Ctrl+C leave" + } + v := m.run.View() + if i := strings.Index(v, "\n\n"); i >= 0 { + v = v[i+2:] // init's own title; the frame has one + } + keys := "Ctrl+C stop" + if m.finished { + keys = "Enter or Q leave · PgUp/PgDn scroll" + if m.run.err == nil { + v += "\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n" + } + } + return "Install", v, keys } func (m setupModel) welcomeView() string { var b strings.Builder - fmt.Fprintf(&b, "Nothing of llmgw is installed on %s (%s).\n\n", m.host, strings.Join(m.ips, ", ")) + fmt.Fprintf(&b, "\n Nothing of llmgw is installed on %s.\n\n", lipgloss.NewStyle().Bold(true).Render(m.host)) opt := func(i int, title, detail string) { mark := "( )" if m.choice == i { mark = "(" + sOK.Render("•") + ")" - title = sTitle.Render(title) + title = fSelected.Render(title) } fmt.Fprintf(&b, " %s %s\n %s\n\n", mark, title, sDim.Render(detail)) } - opt(0, "One machine", "this machine runs everything: the control plane, a model mirror, and models if it has room.") + opt(0, "One machine", "This machine runs everything: control plane, model mirror,\n and models if it has room.") opt(1, "High availability — three machines", - "this one is the control plane; two more keep the store, replicated, so losing any one loses nothing.") - b.WriteString(sDim.Render("↑↓ to choose · enter to go on · ctrl+c to leave") + "\n") + "This one is the control plane; two more keep the store, replicated,\n so losing any one machine loses nothing.") return b.String() } @@ -706,104 +746,114 @@ func (m setupModel) adminView() string { var b strings.Builder label := func(i int, s string) string { if m.afocus == i { - return sTitle.Render("› " + s) + return fSelected.Render("› " + s) } return " " + s } + b.WriteString("\n") fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(0, "Administrator's email"), m.admin.View(), sDim.Render("A password is generated and shown once, at the end.")) fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(1, "Other names nodes will reach this machine by"), m.names.View(), - sDim.Render("Already known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address.")) - b.WriteString(label(2, lipgloss.NewStyle().Bold(true).Render("Next")) + "\n\n") - b.WriteString(sDim.Render("tab or ↑↓ to move · enter on Next · esc back") + "\n") + sDim.Render("Known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address.")) + b.WriteString(button(m.afocus == 2, "Next") + "\n") return b.String() } +// button is a [ Label ] that shows when it has the focus. +func button(focused bool, label string) string { + if focused { + return " " + fBar.Render(" "+label+" ") + } + return " " + lipgloss.NewStyle().Bold(true).Render("[ "+label+" ]") +} + func (m setupModel) machinesView() string { var b strings.Builder - b.WriteString("The two machines that keep the store. They need ssh, a user that can sudo, and\n" + - "Ubuntu 22.04/24.04 or Debian 12; PostgreSQL, Patroni and etcd are installed on them.\n\n") + b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" + + " PostgreSQL, Patroni and etcd are installed on them.\n\n") for i, d := range m.db { base := i * mfCount // Padded before it is coloured: the colour's escape codes count // toward a width and threw the focused row out of line. label := func(f int, s string) string { - s = fmt.Sprintf("%-16s", s) + s = fmt.Sprintf("%-15s", s) if m.mfocus == base+f { - return sTitle.Render("› " + s) + return fSelected.Render("› " + s) } return " " + s } - fmt.Fprintf(&b, "%s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))) + fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))) if d.visible(mfAddr) { - fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View()) - fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View()) + fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View()) + fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View()) } var auth []string for a := 0; a < authCount; a++ { if a == d.auth { - auth = append(auth, sOK.Render("["+authNames[a]+"]")) + auth = append(auth, "("+sOK.Render("•")+") "+authNames[a]) } else { - auth = append(auth, sDim.Render(" "+authNames[a]+" ")) + auth = append(auth, "( ) "+sDim.Render(authNames[a])) } } - fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "log in with"), strings.Join(auth, " ")) + fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "Log in with"), strings.Join(auth, " ")) if d.visible(mfSecret) { - fmt.Fprintf(&b, " %s %s\n", label(mfSecret, authNames[d.auth]), d.secret.View()) + fmt.Fprintf(&b, " %s %s\n", label(mfSecret, strings.ToUpper(authNames[d.auth][:1])+authNames[d.auth][1:]), d.secret.View()) } if d.visible(mfSudo) { - fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "sudo password"), d.sudo.View()) + fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "Sudo password"), d.sudo.View()) } if d.auth == authJoin { - b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next, and dials out to this machine.") + "\n") + b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next.") + "\n") } b.WriteString("\n") } - btn := lipgloss.NewStyle().Bold(true).Render("Check the three machines") - if m.mfocus == 2*mfCount { - btn = sTitle.Render("› " + btn) - } else { - btn = " " + btn - } - b.WriteString(btn + "\n\n") - b.WriteString(sDim.Render("tab or ↑↓ to move · ←→ to choose how to log in · passwords are kept in memory only · esc back") + "\n") + b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n") + b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n") return b.String() } func (m setupModel) waitView() string { var b strings.Builder - fmt.Fprintf(&b, "On each machine below, as root. It dials this machine on port %d; nothing dials in.\n\n", joinpkg.DefaultPort) + fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort) for i, d := range m.db { if d.auth != authJoin { continue } title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)) if d.joined != nil { - fmt.Fprintf(&b, "%s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr) + fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr) continue } - fmt.Fprintf(&b, "%s %s waiting\n", title, m.spin.View()) + fmt.Fprintf(&b, " %s %s waiting\n", title, m.spin.View()) b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n") for _, l := range joinInstall() { - b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n") + b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n") } - b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n") + b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n") } - b.WriteString(sDim.Render("Each code is for one machine and good for an hour. esc back · ctrl+c to leave") + "\n") return b.String() } func (m setupModel) checkView() string { var b strings.Builder + b.WriteString("\n") if m.checking { b.WriteString(" " + m.spin.View() + " checking the three machines…\n") return b.String() } + switch m.report.Worst() { + case preflight.Fail: + b.WriteString(" " + sBad.Render("✗ Fix what is marked ✗, then check again (R).") + "\n") + case preflight.Warn: + b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n") + default: + b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n") + } machine := "" for _, f := range m.report.Findings { if f.Machine != machine { machine = f.Machine - b.WriteString("\n" + lipgloss.NewStyle().Bold(true).Render(machine) + "\n") + b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render(machine) + "\n") } mark := sOK.Render("✓") switch f.Level { @@ -812,51 +862,45 @@ func (m setupModel) checkView() string { case preflight.Fail: mark = sBad.Render("✗") } - line := fmt.Sprintf(" %s %-13s %s", mark, f.Check, truncate(f.Detail, max(20, m.width-22))) + fmt.Fprintf(&b, " %s %-13s %s\n", mark, f.Check, f.Detail) if f.Level != preflight.OK && f.Fix != "" { - line += "\n " + sDim.Render(truncate(f.Fix, max(20, m.width-20))) + b.WriteString(" " + sDim.Render(f.Fix) + "\n") } - b.WriteString(line + "\n") - } - b.WriteString("\n") - switch m.report.Worst() { - case preflight.Fail: - b.WriteString(sBad.Render("Fix what is marked ✗, then check again.") + "\n" + - sDim.Render("r to check again · esc to change the machines") + "\n") - case preflight.Warn: - b.WriteString(sWarn.Render("It can go ahead; what is marked ! is worth a look.") + "\n" + - sDim.Render("enter to see the plan · r to check again · esc back") + "\n") - default: - b.WriteString(sOK.Render("All three are ready.") + "\n" + sDim.Render("enter to see the plan · esc back") + "\n") } return b.String() } func (m setupModel) planView() string { var b strings.Builder - b.WriteString("What will be installed:\n\n") + b.WriteString("\n What will be installed:\n\n") fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), - "control plane and gateway, model mirror, etcd (the third vote), administrator "+strings.TrimSpace(m.admin.Value())) + "control plane and gateway, model mirror, etcd (the third vote)") + fmt.Fprintf(&b, " administrator %s\n", strings.TrimSpace(m.admin.Value())) granted := false for _, d := range m.db { - fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(strings.TrimSpace(d.addr.Value())), - "PostgreSQL under Patroni, etcd") + addr := strings.TrimSpace(d.addr.Value()) + if d.joined != nil { + addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined") + } + fmt.Fprintf(&b, " %s\n PostgreSQL under Patroni, etcd\n", lipgloss.NewStyle().Bold(true).Render(addr)) granted = granted || d.sudo.Value() != "" } - b.WriteString("\n The store is replicated from the first to the second; if the primary is lost the\n" + - " standby takes over in about thirty seconds, and the control plane follows it.\n") + b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" + + " the standby takes over in about thirty seconds, and the control plane follows.\n") if granted { - b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the install\n finishes, and not after.") + "\n") + b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the\n install finishes, and not after.") + "\n") + } + b.WriteString("\n " + sDim.Render("The same from a script, with ssh keys in place:") + "\n") + for _, l := range strings.Split(m.command(), "\n") { + b.WriteString(" " + l + "\n") } - b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n") for _, d := range m.db { if d.joined != nil { - b.WriteString(sDim.Render("(a machine that joined with a code needs ssh for that: the code is for these screens)") + "\n") + b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n") break } } - b.WriteString("\n") - b.WriteString(lipgloss.NewStyle().Bold(true).Render("enter to install") + sDim.Render(" · esc back · ctrl+c to leave") + "\n") + b.WriteString("\n" + button(true, "Enter to install") + "\n") return b.String() } @@ -871,7 +915,7 @@ func runSetupTUI(ctx context.Context, out io.Writer) error { } } m := newSetupModel(host, ips) - p := tea.NewProgram(m, tea.WithOutput(out)) + p := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()) m.ref.p = p final, err := p.Run() if err != nil { diff --git a/internal/cli/tui_setup_test.go b/internal/cli/tui_setup_test.go index 5f77820..e63e6bd 100644 --- a/internal/cli/tui_setup_test.go +++ b/internal/cli/tui_setup_test.go @@ -43,8 +43,12 @@ func TestTheSetupAsksForWhatHighAvailabilityNeedsAndRefusesWhatWillNotDo(t *test if s := m.(setupModel); s.page != pAdmin || !strings.Contains(s.formErr, "email") { t.Fatalf("went on without an administrator: %+v", s.formErr) } - // The refusal puts the cursor back on the email. - m = press(m, "a@example.com", "tab", "tab", "enter") + // The refusal is a dialog; a key closes it, and the cursor is back on + // the email. + if !strings.Contains(m.View(), "Not yet") { + t.Error("the refusal was not shown as a dialog") + } + m = press(m, "x", "a@example.com", "tab", "tab", "enter") s := m.(setupModel) if s.page != pMachines { t.Fatalf("page %d: %s", s.page, s.formErr) @@ -117,3 +121,18 @@ func TestTheConsoleOffersToReplicateOnlyAStoreThatIsNot(t *testing.T) { } _ = adminapi.Summary{} } + +func TestTheSetupIsDrawnAsWindows(t *testing.T) { + m := newSetupModel("ctl", []string{"10.0.0.5"}) + m.width, m.height = 100, 30 + v := m.View() + lines := strings.Split(v, "\n") + if len(lines) != 30 { + t.Errorf("%d lines for a 30-line terminal", len(lines)) + } + for _, want := range []string{"llmgw setup", "Steps", "Start", "╭", "One machine", "Ctrl+C"} { + if !strings.Contains(v, want) { + t.Errorf("the screen lacks %q:\n%s", want, v) + } + } +} diff --git a/internal/control/agentsvc/controlplanes.go b/internal/control/agentsvc/controlplanes.go new file mode 100644 index 0000000..fbfbee7 --- /dev/null +++ b/internal/control/agentsvc/controlplanes.go @@ -0,0 +1,65 @@ +package agentsvc + +import ( + "context" + "slices" + "time" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" +) + +// SetControlPlanes gives the service where the list of control planes comes +// from. Every agent is told it when it registers, and again when it changes. +func (s *Service) SetControlPlanes(list func(context.Context) ([]string, error)) { + s.mu.Lock() + s.controlPlanes = list + s.mu.Unlock() +} + +// controlPlanesMsg is the list as an agent is sent it, or nil when there is +// none to send: a cluster with no list recorded keeps what its agents were +// configured with. +func (s *Service) controlPlanesMsg(ctx context.Context) *llmgwv1.ControlMessage { + s.mu.RLock() + list := s.controlPlanes + s.mu.RUnlock() + if list == nil { + return nil + } + servers, err := list(ctx) + if err != nil || len(servers) == 0 { + return nil + } + return &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_ControlPlanes{ + ControlPlanes: &llmgwv1.ControlPlanes{Servers: servers}}} +} + +// WatchControlPlanes tells every connected agent the list when it changes, +// checking every so often. A control plane added by `llmgw server add` reaches +// the nodes within that time, without their being touched. +func (s *Service) WatchControlPlanes(ctx context.Context, every time.Duration) { + var last []string + t := time.NewTicker(every) + defer t.Stop() + for { + if msg := s.controlPlanesMsg(ctx); msg != nil { + now := msg.GetControlPlanes().GetServers() + if !slices.Equal(now, last) { + for _, node := range s.ConnectedNodes() { + if err := s.send(node, msg); err != nil { + s.log.Warn("could not tell a node the control planes", "node", node, "err", err) + } + } + if last != nil { + s.log.Info("told the nodes the control planes changed", "servers", now) + } + last = now + } + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index d939ef4..2813955 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -61,6 +61,9 @@ type Service struct { choices func(nodeID string) map[string]string installableBy map[string][]string + // controlPlanes lists what agents may dial; see SetControlPlanes. + controlPlanes func(context.Context) ([]string, error) + // logWaits are the log requests awaiting a reply, by request id. logMu sync.Mutex logWaits map[string]chan *llmgwv1.LogsReply @@ -229,6 +232,13 @@ func (s *Service) Connect( return err } } + // Which control planes there are, so an agent configured with one learns + // of a second added since. + if msg := s.controlPlanesMsg(ctx); msg != nil { + if err := stream.Send(msg); err != nil { + return err + } + } // Recover replica state from what this node is already running, before any // reconciliation pass can conclude the cluster is empty and place diff --git a/internal/pgha/config.go b/internal/pgha/config.go index bd4d284..d8edf3c 100644 --- a/internal/pgha/config.go +++ b/internal/pgha/config.go @@ -22,6 +22,9 @@ const Scope = "llmgw" // control plane reads the cluster's state from. const PatroniPort = 8008 +// restPort is where this package asks Patroni; a test moves it. +var restPort = PatroniPort + // Role and Database are what the control plane connects as. const ( Role = "llmgw" @@ -83,6 +86,10 @@ func Patroni(m Member, db, etcd []Member, clients []string, version string, s Se seen[c] = true hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c)) } + var hbaLines string + for _, h := range hba { + hbaLines += " - " + h + "\n" + } var b strings.Builder fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd. scope: %[1]s @@ -113,7 +120,11 @@ bootstrap: wal_keep_size: 1GB wal_log_hints: "on" password_encryption: scram-sha-256 - initdb: + # In the cluster's shared configuration rather than each member's, so a + # control plane added later is let in by one change through Patroni's + # API (llmgw server add) instead of an edit on every member. + pg_hba: +%[9]s initdb: - encoding: UTF8 - data-checksums @@ -135,11 +146,7 @@ postgresql: password: "%[7]s" parameters: unix_socket_directories: /var/run/postgresql - pg_hba: -`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator) - for _, h := range hba { - fmt.Fprintf(&b, " - %s\n", h) - } +`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines) b.WriteString(` tags: nofailover: false diff --git a/internal/pgha/hba.go b/internal/pgha/hba.go new file mode 100644 index 0000000..3a4243b --- /dev/null +++ b/internal/pgha/hba.go @@ -0,0 +1,91 @@ +package pgha + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "slices" + "strconv" + "time" +) + +// ClientLine is the access rule that lets a control plane at ip in. +func ClientLine(ip string) string { + return fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, ip) +} + +// AllowClient lets a control plane at ip into the cluster's database: one +// change to the cluster's shared configuration through Patroni's API, which +// every member applies. It says whether anything changed. +// +// A cluster set up before the rules moved into the shared configuration +// keeps them in each member's own file, which this cannot reach; that is +// said, with what to add, rather than half done. +func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) { + client := &http.Client{Timeout: 5 * time.Second} + var lastErr error + for _, h := range hosts { + base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort)) + cfg, err := getConfig(ctx, client, base) + if err != nil { + lastErr = err + continue + } + var hba []string + if pg, ok := cfg["postgresql"].(map[string]any); ok { + if list, ok := pg["pg_hba"].([]any); ok { + for _, l := range list { + if s, ok := l.(string); ok { + hba = append(hba, s) + } + } + } + } + if len(hba) == 0 { + return false, fmt.Errorf("this cluster keeps its access rules in each member's "+ + "/etc/patroni/config.yml, not in its shared configuration: add %q to pg_hba there "+ + "on every member, and run `patronictl reload llmgw`", ClientLine(ip)) + } + line := ClientLine(ip) + if slices.Contains(hba, line) { + return false, nil + } + body, _ := json.Marshal(map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}}) + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(body)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + lastErr = err + continue + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("Patroni on %s refused the change: %s", h, resp.Status) + } + return true, nil + } + return false, fmt.Errorf("no member's Patroni answered: %w", lastErr) +} + +func getConfig(ctx context.Context, client *http.Client, base string) (map[string]any, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/config", nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var cfg map[string]any + if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil { + return nil, fmt.Errorf("%s/config: %w", base, err) + } + return cfg, nil +} diff --git a/internal/pgha/pgha_test.go b/internal/pgha/pgha_test.go index a2bc291..b3029fe 100644 --- a/internal/pgha/pgha_test.go +++ b/internal/pgha/pgha_test.go @@ -3,6 +3,10 @@ package pgha import ( "context" "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" "strings" "sync" "testing" @@ -10,6 +14,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/expfmt" + "go.yaml.in/yaml/v3" ) var ( @@ -208,3 +213,70 @@ func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) { t.Errorf("%s", cl) } } + +func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) { + // Where `server add` can change them for every member at once. + cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"}) + var doc struct { + Bootstrap struct { + DCS struct { + PostgreSQL struct { + PgHBA []string `yaml:"pg_hba"` + } `yaml:"postgresql"` + } `yaml:"dcs"` + } `yaml:"bootstrap"` + PostgreSQL map[string]any `yaml:"postgresql"` + } + if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil { + t.Fatalf("not YAML: %v\n%s", err, cfg) + } + hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA + if len(hba) < 5 || hba[0] != "local all all peer" { + t.Errorf("shared pg_hba: %q", hba) + } + if _, local := doc.PostgreSQL["pg_hba"]; local { + t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one") + } +} + +func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) { + var patched []string + hba := []any{"local all all peer", ClientLine("10.0.0.140")} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + _ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}}) + case http.MethodPatch: + var body struct { + PostgreSQL struct { + PgHBA []string `json:"pg_hba"` + } `json:"postgresql"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + patched = body.PostgreSQL.PgHBA + } + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + old := restPort + restPort, _ = strconv.Atoi(u.Port()) + defer func() { restPort = old }() + + changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150") + if err != nil || !changed { + t.Fatalf("%v %v", changed, err) + } + // Everything that was there is kept, and the new line added once. + if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") { + t.Errorf("patched %q", patched) + } + if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed { + t.Errorf("one already let in was changed again: %v %v", changed, err) + } + + hba = nil // the rules are in each member's file + if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil || + !strings.Contains(err.Error(), "each member") { + t.Errorf("rules kept per member: %v", err) + } +} diff --git a/internal/pgha/status.go b/internal/pgha/status.go index 74c1b16..b67f1fc 100644 --- a/internal/pgha/status.go +++ b/internal/pgha/status.go @@ -73,7 +73,7 @@ func HTTPStatus(ctx context.Context, hosts []string) (Cluster, error) { client := &http.Client{Timeout: 3 * time.Second} var last error for _, h := range hosts { - u := "http://" + net.JoinHostPort(h, strconv.Itoa(PatroniPort)) + "/cluster" + u := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort)) + "/cluster" req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return Cluster{}, err diff --git a/internal/router/handlers.go b/internal/router/handlers.go index b758f11..21e222c 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -123,6 +123,19 @@ func (r *Router) Handler() http.Handler { w.WriteHeader(http.StatusOK) }) + // Leadership: 200 on the control plane that places work, 503 on a + // standby. What a floating address follows (llmgw server add --vip): + // unlike /ready it does not wait for a replica, so a cluster with no + // node yet still has its address on one machine. A process that is not + // a control plane, or one with no rival, is always the leader. + mux.HandleFunc("GET /leader", func(w http.ResponseWriter, _ *http.Request) { + if f := r.isLeader.Load(); f != nil && !(*f)() { + http.Error(w, "this control plane is a standby", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + }) + // The landing page says how to connect and nothing about what is here, so // it is safe to serve without a key — and it has to be, because the person // reading it is the one who does not have one yet. @@ -227,7 +240,7 @@ func (r *Router) Handler() http.Handler { // route must not be able to add an unauthenticated one by omission. Health // stays open because a load balancer has no key. return auth.Middleware(r.auth, r.log, - "/health", "/ready", "/", "/about", "/_static/", "/claim/", + "/health", "/ready", "/leader", "/", "/about", "/_static/", "/claim/", "/signup", "/login", "/logout", "/profile", "/profile/", "/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux) } diff --git a/internal/router/hardware_leader_test.go b/internal/router/hardware_leader_test.go new file mode 100644 index 0000000..b0b685f --- /dev/null +++ b/internal/router/hardware_leader_test.go @@ -0,0 +1,43 @@ +package router + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "testing" +) + +func TestTheLeaderEndpointFollowsLeadership(t *testing.T) { + rt := newTestRouter(t, baseConfig()) + get := func() int { + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/leader", nil)) + return rec.Code + } + // No rival, nothing to follow: the one process leads. + if c := get(); c != http.StatusOK { + t.Errorf("with no leadership: %d", c) + } + leading := false + rt.SetLeader(func() bool { return leading }) + if c := get(); c != http.StatusServiceUnavailable { + t.Errorf("a standby: %d", c) + } + leading = true + if c := get(); c != http.StatusOK { + t.Errorf("the leader: %d", c) + } +} + +func TestTheLeaderEndpointNeedsNoKey(t *testing.T) { + // keepalived and load balancers ask it, and they have no key. + cfg := baseConfig() + cfg.Auth.Enabled = true + cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db") + rt := newTestRouter(t, cfg) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/leader", nil)) + if rec.Code != http.StatusOK { + t.Errorf("/leader without a key: %d", rec.Code) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index bfa0503..e0fae3e 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -83,6 +83,8 @@ type Router struct { auth *auth.Authenticator accounts *account.Service store *storepkg.DB + // isLeader says whether this process leads; nil is always. + isLeader atomic.Pointer[func() bool] acct *accounting.Accountant // threads holds conversations and memory notes. It is store when auth is on, @@ -474,6 +476,9 @@ func (r *Router) Stop() { } } +// SetLeader tells /leader how to know whether this process leads. +func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) } + // Registry exposes the endpoint registry, for tests and admin endpoints. func (r *Router) Registry() *endpoint.Registry { return r.reg } diff --git a/internal/store/controlplanes.go b/internal/store/controlplanes.go new file mode 100644 index 0000000..956948b --- /dev/null +++ b/internal/store/controlplanes.go @@ -0,0 +1,44 @@ +package store + +import ( + "context" + "database/sql" + "time" +) + +// ControlPlane is one control plane agents may dial. +type ControlPlane struct { + // URL is what an agent dials: https://10.0.0.150:7443. + URL string + Name string + AddedAt time.Time +} + +// ControlPlanes lists them, oldest first: the first is the one agents try +// first. +func (s *DB) ControlPlanes(ctx context.Context) ([]ControlPlane, error) { + return queryAll(ctx, s.db, `SELECT url, name, added_at FROM control_planes ORDER BY added_at, url`, nil, + func(rows *sql.Rows) (ControlPlane, error) { + var c ControlPlane + var at int64 + err := rows.Scan(&c.URL, &c.Name, &at) + c.AddedAt = fromMillis(at) + return c, err + }) +} + +// PutControlPlane records one, keeping when it was first added. +func (s *DB) PutControlPlane(ctx context.Context, c ControlPlane) error { + if c.AddedAt.IsZero() { + c.AddedAt = time.Now() + } + _, err := s.db.ExecContext(ctx, `INSERT INTO control_planes (url, name, added_at) VALUES (?, ?, ?) + ON CONFLICT (url) DO UPDATE SET name = excluded.name`, c.URL, c.Name, millis(c.AddedAt)) + return err +} + +// RemoveControlPlane forgets one. +func (s *DB) RemoveControlPlane(ctx context.Context, url string) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM control_planes WHERE url = ?`, url) + return err +} diff --git a/internal/store/copy.go b/internal/store/copy.go index 1c72157..a5f68d1 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -46,6 +46,7 @@ var copyOrder = []string{ // of them at the moment of the move. "cluster_secrets", "node_engines", + "control_planes", } // notCopied are tables deliberately left behind. diff --git a/internal/store/migrations/postgres/0016_control_planes.sql b/internal/store/migrations/postgres/0016_control_planes.sql new file mode 100644 index 0000000..11bacee --- /dev/null +++ b/internal/store/migrations/postgres/0016_control_planes.sql @@ -0,0 +1,10 @@ +-- Ported from the SQLite migration of the same number. +-- The control planes agents may dial, by the address they dial: what the +-- leader tells every agent, so a control plane added after the nodes were +-- onboarded is one they fail over to without their configuration being +-- edited. Written by `llmgw server add` and by init for the first one. +CREATE TABLE control_planes ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + added_at BIGINT NOT NULL +); diff --git a/internal/store/migrations/sqlite/0016_control_planes.sql b/internal/store/migrations/sqlite/0016_control_planes.sql new file mode 100644 index 0000000..df93f2c --- /dev/null +++ b/internal/store/migrations/sqlite/0016_control_planes.sql @@ -0,0 +1,9 @@ +-- The control planes agents may dial, by the address they dial: what the +-- leader tells every agent, so a control plane added after the nodes were +-- onboarded is one they fail over to without their configuration being +-- edited. Written by `llmgw server add` and by init for the first one. +CREATE TABLE control_planes ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + added_at INTEGER NOT NULL +); diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go index 273caf9..77f022f 100644 --- a/internal/store/postgres_test.go +++ b/internal/store/postgres_test.go @@ -43,7 +43,7 @@ func openPG(t *testing.T) *DB { // every subtest gets a fresh file, and failed only here. `TRUNCATE thread_messages, threads, memory_notes, usage, api_keys, memberships, key_claims, limits, models, sessions, users, projects, teams, - organizations, leadership RESTART IDENTITY CASCADE`) + organizations, leadership, control_planes RESTART IDENTITY CASCADE`) if err != nil { t.Fatalf("clearing the test database: %v", err) } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5797582..b807de1 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -493,3 +493,28 @@ func TestNoDefaultsLeavesLimitsUnset(t *testing.T) { t.Errorf("limits appeared with no defaults set: rpm=%v tpm=%v", e.RPM, e.TPM) } } + +func TestControlPlanesAreKeptInTheOrderTheyWereAdded(t *testing.T) { + ctx := context.Background() + s := openDB(t) + at := time.Now() + for i, u := range []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"} { + if err := s.PutControlPlane(ctx, ControlPlane{URL: u, Name: "c", AddedAt: at.Add(time.Duration(i) * time.Second)}); err != nil { + t.Fatal(err) + } + } + // Recorded again, as init and server add do: not moved, not doubled. + if err := s.PutControlPlane(ctx, ControlPlane{URL: "https://10.0.0.140:7443", Name: "ctl"}); err != nil { + t.Fatal(err) + } + cps, err := s.ControlPlanes(ctx) + if err != nil || len(cps) != 2 || cps[0].URL != "https://10.0.0.140:7443" || cps[0].Name != "ctl" { + t.Fatalf("%+v %v", cps, err) + } + if err := s.RemoveControlPlane(ctx, cps[1].URL); err != nil { + t.Fatal(err) + } + if cps, _ := s.ControlPlanes(ctx); len(cps) != 1 { + t.Errorf("%+v", cps) + } +}