From 7245dbe1dd965bbfc8b9b9687288338ecd2f16a3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 7 Oct 2026 18:01:45 +0200 Subject: [PATCH] llmgw stop, start and restart act on the whole system Issue #3 asked for commands to stop and start the system, and 05aa21c gave only one machine's services. Now: - llmgw stop sets serving.stopped in the cluster's settings, live: the reconciler wants every model at no replicas, freeing every GPU, and the gateway answers every POST under /v1/ with 503 system_stopped. The control planes, agents and mirror keep running, and no model's definition changes. - llmgw start clears it; every model is placed again as it asks. - llmgw restart has every agent restart over its stream (RestartAgent), then the other control planes over ssh, standbys before a leader, then this one; engines keep serving. - llmgw status is the cluster's summary, saying when it is stopped. One machine's services moved to llmgw service status|start|stop|restart. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 +- api/gen/llmgw/v1/agent.pb.go | 530 ++++++++++-------- api/proto/llmgw/v1/agent.proto | 7 + docs/admin-api.openapi.json | 51 ++ docs/install.md | 40 +- docs/reference.md | 23 + internal/agent/agent.go | 8 + internal/cli/clustersettings.go | 2 + internal/cli/root.go | 6 +- internal/cli/server.go | 14 + internal/cli/service.go | 19 +- internal/cli/service_test.go | 52 +- internal/cli/status.go | 3 + internal/cli/system.go | 216 +++++++ internal/config/config.go | 4 + internal/config/serving.go | 11 + internal/control/adminapi/api.go | 3 + internal/control/adminapi/routes.go | 3 + internal/control/adminapi/summary.go | 5 +- internal/control/adminapi/system.go | 22 + internal/control/agentsvc/firewall.go | 19 + internal/control/reconciler/demand.go | 4 + internal/control/reconciler/reconciler.go | 14 + .../control/reconciler/reconciler_test.go | 24 + internal/docgen/reference_gen.go | 4 + internal/docgen/render.go | 1 + internal/router/handlers.go | 19 +- internal/router/router.go | 7 +- internal/router/router_test.go | 21 + 29 files changed, 891 insertions(+), 254 deletions(-) create mode 100644 internal/cli/system.go create mode 100644 internal/config/serving.go create mode 100644 internal/control/adminapi/system.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b8723d..9f84a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,10 +28,15 @@ What was left of the field report from v0.10.6. ### Added -- **`llmgw status`, `start`, `stop` and `restart`** for this machine's llmgw - services — server, agent, mirror, or those named — in the order they - depend on each other. Stopping the agent leaves the engines serving unless - `--engines` is given; stopping a leader hands over to a standby. +- **The whole system stops, starts and restarts from one command**: `llmgw + stop` takes every model down and answers every request for one with 503 + (`system_stopped`), keeping the control planes, agents and mirror running + and no model's definition changed; `llmgw start` brings every model back. + It is the cluster's setting `serving.stopped`, live on every control plane. + `llmgw restart` restarts every node's agent and every control plane, the + leader last, with the engines serving throughout. `llmgw status` says + whether the system serves. One machine's services are `llmgw service + status|start|stop|restart`. - **A replica says what it computes on**: `ctl replicas` (ON), the console and the admin API show `CUDA` with the GPUs its processes hold memory on and how much, as nvidia-smi sees them, or `CPU` — for every engine, kept by diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index e66d7c0..5a283c1 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26, 0} } // AgentMessage is anything a node tells the control plane. @@ -390,6 +390,7 @@ type ControlMessage struct { // *ControlMessage_Upgrade // *ControlMessage_ControlPlanes // *ControlMessage_Firewall + // *ControlMessage_RestartAgent Msg isControlMessage_Msg `protobuf_oneof:"msg"` // trace is the W3C trace context (traceparent, and tracestate when there // is one) of the control plane's span that sent this command, so what the @@ -546,6 +547,15 @@ func (x *ControlMessage) GetFirewall() *Firewall { return nil } +func (x *ControlMessage) GetRestartAgent() *RestartAgent { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_RestartAgent); ok { + return x.RestartAgent + } + } + return nil +} + func (x *ControlMessage) GetTrace() map[string]string { if x != nil { return x.Trace @@ -605,6 +615,10 @@ type ControlMessage_Firewall struct { Firewall *Firewall `protobuf:"bytes,13,opt,name=firewall,proto3,oneof"` } +type ControlMessage_RestartAgent struct { + RestartAgent *RestartAgent `protobuf:"bytes,14,opt,name=restart_agent,json=restartAgent,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -629,6 +643,54 @@ func (*ControlMessage_ControlPlanes) isControlMessage_Msg() {} func (*ControlMessage_Firewall) isControlMessage_Msg() {} +func (*ControlMessage_RestartAgent) isControlMessage_Msg() {} + +// RestartAgent has the agent start again, as llmgw restart asks of every +// node: its engines keep serving and are taken over again. +type RestartAgent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Reason string `protobuf:"bytes,1,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RestartAgent) Reset() { + *x = RestartAgent{} + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RestartAgent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RestartAgent) ProtoMessage() {} + +func (x *RestartAgent) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RestartAgent.ProtoReflect.Descriptor instead. +func (*RestartAgent) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *RestartAgent) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + // Firewall is whether llmgw's firewall is kept on the node — its nftables // table, which lets the cluster's networks alone reach the engines' ports and // the node exporter — sent when an agent registers and whenever the @@ -649,7 +711,7 @@ type Firewall struct { func (x *Firewall) Reset() { *x = Firewall{} - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -661,7 +723,7 @@ func (x *Firewall) String() string { func (*Firewall) ProtoMessage() {} func (x *Firewall) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -674,7 +736,7 @@ func (x *Firewall) ProtoReflect() protoreflect.Message { // Deprecated: Use Firewall.ProtoReflect.Descriptor instead. func (*Firewall) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} } func (x *Firewall) GetEnabled() bool { @@ -718,7 +780,7 @@ type ControlPlanes struct { func (x *ControlPlanes) Reset() { *x = ControlPlanes{} - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -730,7 +792,7 @@ func (x *ControlPlanes) String() string { func (*ControlPlanes) ProtoMessage() {} func (x *ControlPlanes) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -743,7 +805,7 @@ func (x *ControlPlanes) ProtoReflect() protoreflect.Message { // Deprecated: Use ControlPlanes.ProtoReflect.Descriptor instead. func (*ControlPlanes) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} } func (x *ControlPlanes) GetServers() []string { @@ -843,7 +905,7 @@ type Register struct { func (x *Register) Reset() { *x = Register{} - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -855,7 +917,7 @@ func (x *Register) String() string { func (*Register) ProtoMessage() {} func (x *Register) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -868,7 +930,7 @@ func (x *Register) ProtoReflect() protoreflect.Message { // Deprecated: Use Register.ProtoReflect.Descriptor instead. func (*Register) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} } func (x *Register) GetNodeId() string { @@ -1020,7 +1082,7 @@ type Host struct { func (x *Host) Reset() { *x = Host{} - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1032,7 +1094,7 @@ func (x *Host) String() string { func (*Host) ProtoMessage() {} func (x *Host) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1045,7 +1107,7 @@ func (x *Host) ProtoReflect() protoreflect.Message { // Deprecated: Use Host.ProtoReflect.Descriptor instead. func (*Host) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} } func (x *Host) GetOsId() string { @@ -1112,7 +1174,7 @@ type Storage struct { func (x *Storage) Reset() { *x = Storage{} - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1124,7 +1186,7 @@ func (x *Storage) String() string { func (*Storage) ProtoMessage() {} func (x *Storage) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1137,7 +1199,7 @@ func (x *Storage) ProtoReflect() protoreflect.Message { // Deprecated: Use Storage.ProtoReflect.Descriptor instead. func (*Storage) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} } func (x *Storage) GetPath() string { @@ -1208,7 +1270,7 @@ type RunningShard struct { func (x *RunningShard) Reset() { *x = RunningShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1220,7 +1282,7 @@ func (x *RunningShard) String() string { func (*RunningShard) ProtoMessage() {} func (x *RunningShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1233,7 +1295,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message { // Deprecated: Use RunningShard.ProtoReflect.Descriptor instead. func (*RunningShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *RunningShard) GetReplicaId() string { @@ -1393,7 +1455,7 @@ type GPU struct { func (x *GPU) Reset() { *x = GPU{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1405,7 +1467,7 @@ func (x *GPU) String() string { func (*GPU) ProtoMessage() {} func (x *GPU) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1418,7 +1480,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message { // Deprecated: Use GPU.ProtoReflect.Descriptor instead. func (*GPU) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} } func (x *GPU) GetIndex() int32 { @@ -1594,7 +1656,7 @@ type MIGDevice struct { func (x *MIGDevice) Reset() { *x = MIGDevice{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1606,7 +1668,7 @@ func (x *MIGDevice) String() string { func (*MIGDevice) ProtoMessage() {} func (x *MIGDevice) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1619,7 +1681,7 @@ func (x *MIGDevice) ProtoReflect() protoreflect.Message { // Deprecated: Use MIGDevice.ProtoReflect.Descriptor instead. func (*MIGDevice) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *MIGDevice) GetParentIndex() int32 { @@ -1685,7 +1747,7 @@ type FabricAttachment struct { func (x *FabricAttachment) Reset() { *x = FabricAttachment{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1697,7 +1759,7 @@ func (x *FabricAttachment) String() string { func (*FabricAttachment) ProtoMessage() {} func (x *FabricAttachment) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1710,7 +1772,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead. func (*FabricAttachment) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *FabricAttachment) GetFabric() string { @@ -1752,7 +1814,7 @@ type LocalWeights struct { func (x *LocalWeights) Reset() { *x = LocalWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1764,7 +1826,7 @@ func (x *LocalWeights) String() string { func (*LocalWeights) ProtoMessage() {} func (x *LocalWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1777,7 +1839,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. func (*LocalWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *LocalWeights) GetPath() string { @@ -1833,7 +1895,7 @@ type NodeLoad struct { func (x *NodeLoad) Reset() { *x = NodeLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1845,7 +1907,7 @@ func (x *NodeLoad) String() string { func (*NodeLoad) ProtoMessage() {} func (x *NodeLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1858,7 +1920,7 @@ func (x *NodeLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeLoad.ProtoReflect.Descriptor instead. func (*NodeLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *NodeLoad) GetCpuPercent() float64 { @@ -1991,7 +2053,7 @@ type GPUSensor struct { func (x *GPUSensor) Reset() { *x = GPUSensor{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2003,7 +2065,7 @@ func (x *GPUSensor) String() string { func (*GPUSensor) ProtoMessage() {} func (x *GPUSensor) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2016,7 +2078,7 @@ func (x *GPUSensor) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUSensor.ProtoReflect.Descriptor instead. func (*GPUSensor) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *GPUSensor) GetIndex() int32 { @@ -2230,7 +2292,7 @@ type ShardLoad struct { func (x *ShardLoad) Reset() { *x = ShardLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2242,7 +2304,7 @@ func (x *ShardLoad) String() string { func (*ShardLoad) ProtoMessage() {} func (x *ShardLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2255,7 +2317,7 @@ func (x *ShardLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead. func (*ShardLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *ShardLoad) GetShardId() string { @@ -2323,7 +2385,7 @@ type IBPort struct { func (x *IBPort) Reset() { *x = IBPort{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2335,7 +2397,7 @@ func (x *IBPort) String() string { func (*IBPort) ProtoMessage() {} func (x *IBPort) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2348,7 +2410,7 @@ func (x *IBPort) ProtoReflect() protoreflect.Message { // Deprecated: Use IBPort.ProtoReflect.Descriptor instead. func (*IBPort) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *IBPort) GetHca() string { @@ -2491,7 +2553,7 @@ type SubnetManager struct { func (x *SubnetManager) Reset() { *x = SubnetManager{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2503,7 +2565,7 @@ func (x *SubnetManager) String() string { func (*SubnetManager) ProtoMessage() {} func (x *SubnetManager) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2516,7 +2578,7 @@ func (x *SubnetManager) ProtoReflect() protoreflect.Message { // Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead. func (*SubnetManager) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *SubnetManager) GetGuid() string { @@ -2559,7 +2621,7 @@ type IMEX struct { func (x *IMEX) Reset() { *x = IMEX{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2571,7 +2633,7 @@ func (x *IMEX) String() string { func (*IMEX) ProtoMessage() {} func (x *IMEX) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2584,7 +2646,7 @@ func (x *IMEX) ProtoReflect() protoreflect.Message { // Deprecated: Use IMEX.ProtoReflect.Descriptor instead. func (*IMEX) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *IMEX) GetState() string { @@ -2616,7 +2678,7 @@ type GPUFabricState struct { func (x *GPUFabricState) Reset() { *x = GPUFabricState{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2628,7 +2690,7 @@ func (x *GPUFabricState) String() string { func (*GPUFabricState) ProtoMessage() {} func (x *GPUFabricState) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2641,7 +2703,7 @@ func (x *GPUFabricState) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead. func (*GPUFabricState) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *GPUFabricState) GetIndex() int32 { @@ -2756,7 +2818,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2768,7 +2830,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2781,7 +2843,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *Heartbeat) GetNodeId() string { @@ -2936,7 +2998,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2948,7 +3010,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2961,7 +3023,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *Provision) GetEngines() []*EngineWant { @@ -2994,7 +3056,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3006,7 +3068,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3019,7 +3081,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *EngineWant) GetEngine() string { @@ -3073,7 +3135,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3085,7 +3147,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3098,7 +3160,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *ProvisionStatus) GetEngine() string { @@ -3153,7 +3215,7 @@ type Compute struct { func (x *Compute) Reset() { *x = Compute{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3165,7 +3227,7 @@ func (x *Compute) String() string { func (*Compute) ProtoMessage() {} func (x *Compute) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3178,7 +3240,7 @@ func (x *Compute) ProtoReflect() protoreflect.Message { // Deprecated: Use Compute.ProtoReflect.Descriptor instead. func (*Compute) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *Compute) GetBackend() string { @@ -3238,7 +3300,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3250,7 +3312,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3263,7 +3325,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} } func (x *ShardStatus) GetNodeId() string { @@ -3344,7 +3406,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3356,7 +3418,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3369,7 +3431,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} } func (x *FetchProgress) GetSource() string { @@ -3432,7 +3494,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3444,7 +3506,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3457,7 +3519,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} } func (x *StartShard) GetReplicaId() string { @@ -3604,7 +3666,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3616,7 +3678,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3629,7 +3691,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} } func (x *EngineProfile) GetName() string { @@ -3739,7 +3801,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3751,7 +3813,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3764,7 +3826,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} } func (x *RemoveWeights) GetPaths() []string { @@ -3784,7 +3846,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3796,7 +3858,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3809,7 +3871,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} } func (x *StopShard) GetShardId() string { @@ -3829,7 +3891,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3841,7 +3903,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3854,7 +3916,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} } func (x *DrainShard) GetShardId() string { @@ -3876,7 +3938,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3888,7 +3950,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3901,7 +3963,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} } func (x *Ack) GetMessageId() string { @@ -3945,7 +4007,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3957,7 +4019,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3970,7 +4032,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} } func (x *LogsRequest) GetRequestId() string { @@ -4010,7 +4072,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4022,7 +4084,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4035,7 +4097,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} } func (x *LogsReply) GetRequestId() string { @@ -4081,7 +4143,7 @@ type FabricScanRequest struct { func (x *FabricScanRequest) Reset() { *x = FabricScanRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[35] + mi := &file_llmgw_v1_agent_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4093,7 +4155,7 @@ func (x *FabricScanRequest) String() string { func (*FabricScanRequest) ProtoMessage() {} func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[35] + mi := &file_llmgw_v1_agent_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4106,7 +4168,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead. func (*FabricScanRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} } func (x *FabricScanRequest) GetRequestId() string { @@ -4145,7 +4207,7 @@ type FabricScanReply struct { func (x *FabricScanReply) Reset() { *x = FabricScanReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[36] + mi := &file_llmgw_v1_agent_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4157,7 +4219,7 @@ func (x *FabricScanReply) String() string { func (*FabricScanReply) ProtoMessage() {} func (x *FabricScanReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[36] + mi := &file_llmgw_v1_agent_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4170,7 +4232,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead. func (*FabricScanReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{37} } func (x *FabricScanReply) GetRequestId() string { @@ -4226,7 +4288,7 @@ type FabricTestRequest struct { func (x *FabricTestRequest) Reset() { *x = FabricTestRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[37] + mi := &file_llmgw_v1_agent_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4238,7 +4300,7 @@ func (x *FabricTestRequest) String() string { func (*FabricTestRequest) ProtoMessage() {} func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[37] + mi := &file_llmgw_v1_agent_proto_msgTypes[38] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4251,7 +4313,7 @@ func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead. func (*FabricTestRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{37} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{38} } func (x *FabricTestRequest) GetRequestId() string { @@ -4320,7 +4382,7 @@ type FabricTestReply struct { func (x *FabricTestReply) Reset() { *x = FabricTestReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[38] + mi := &file_llmgw_v1_agent_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4332,7 +4394,7 @@ func (x *FabricTestReply) String() string { func (*FabricTestReply) ProtoMessage() {} func (x *FabricTestReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[38] + mi := &file_llmgw_v1_agent_proto_msgTypes[39] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4345,7 +4407,7 @@ func (x *FabricTestReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead. func (*FabricTestReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{38} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{39} } func (x *FabricTestReply) GetRequestId() string { @@ -4405,7 +4467,7 @@ type UpgradeAgent struct { func (x *UpgradeAgent) Reset() { *x = UpgradeAgent{} - mi := &file_llmgw_v1_agent_proto_msgTypes[39] + mi := &file_llmgw_v1_agent_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4417,7 +4479,7 @@ func (x *UpgradeAgent) String() string { func (*UpgradeAgent) ProtoMessage() {} func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[39] + mi := &file_llmgw_v1_agent_proto_msgTypes[40] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4430,7 +4492,7 @@ func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead. func (*UpgradeAgent) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{39} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{40} } func (x *UpgradeAgent) GetRequestId() string { @@ -4467,7 +4529,7 @@ type UpgradeReply struct { func (x *UpgradeReply) Reset() { *x = UpgradeReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[40] + mi := &file_llmgw_v1_agent_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4479,7 +4541,7 @@ func (x *UpgradeReply) String() string { func (*UpgradeReply) ProtoMessage() {} func (x *UpgradeReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[40] + mi := &file_llmgw_v1_agent_proto_msgTypes[41] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4492,7 +4554,7 @@ func (x *UpgradeReply) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead. func (*UpgradeReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{40} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{41} } func (x *UpgradeReply) GetRequestId() string { @@ -4528,7 +4590,7 @@ type LoraAdapter struct { func (x *LoraAdapter) Reset() { *x = LoraAdapter{} - mi := &file_llmgw_v1_agent_proto_msgTypes[41] + mi := &file_llmgw_v1_agent_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4540,7 +4602,7 @@ func (x *LoraAdapter) String() string { func (*LoraAdapter) ProtoMessage() {} func (x *LoraAdapter) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[41] + mi := &file_llmgw_v1_agent_proto_msgTypes[42] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4553,7 +4615,7 @@ func (x *LoraAdapter) ProtoReflect() protoreflect.Message { // Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead. func (*LoraAdapter) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{41} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{42} } func (x *LoraAdapter) GetName() string { @@ -4596,7 +4658,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "clearFirst\x12\x16\n" + "\x06stored\x18\x04 \x03(\x06R\x06stored\x12\x18\n" + "\aremoved\x18\x05 \x03(\x06R\aremoved\x12\x18\n" + - "\aproblem\x18\x06 \x01(\tR\aproblem\"\x82\x06\n" + + "\aproblem\x18\x06 \x01(\tR\aproblem\"\xc1\x06\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + @@ -4612,13 +4674,16 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aupgrade\x18\n" + " \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgrade\x12@\n" + "\x0econtrol_planes\x18\v \x01(\v2\x17.llmgw.v1.ControlPlanesH\x00R\rcontrolPlanes\x120\n" + - "\bfirewall\x18\r \x01(\v2\x12.llmgw.v1.FirewallH\x00R\bfirewall\x129\n" + + "\bfirewall\x18\r \x01(\v2\x12.llmgw.v1.FirewallH\x00R\bfirewall\x12=\n" + + "\rrestart_agent\x18\x0e \x01(\v2\x16.llmgw.v1.RestartAgentH\x00R\frestartAgent\x129\n" + "\x05trace\x18\f \x03(\v2#.llmgw.v1.ControlMessage.TraceEntryR\x05trace\x1a8\n" + "\n" + "TraceEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x05\n" + - "\x03msg\"\x94\x01\n" + + "\x03msg\"&\n" + + "\fRestartAgent\x12\x16\n" + + "\x06reason\x18\x01 \x01(\tR\x06reason\"\x94\x01\n" + "\bFirewall\x12\x18\n" + "\aenabled\x18\x01 \x01(\bR\aenabled\x12\x1a\n" + "\bnetworks\x18\x02 \x03(\tR\bnetworks\x12*\n" + @@ -5034,123 +5099,125 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 50) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 51) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage (*KVEvents)(nil), // 2: llmgw.v1.KVEvents (*ControlMessage)(nil), // 3: llmgw.v1.ControlMessage - (*Firewall)(nil), // 4: llmgw.v1.Firewall - (*ControlPlanes)(nil), // 5: llmgw.v1.ControlPlanes - (*Register)(nil), // 6: llmgw.v1.Register - (*Host)(nil), // 7: llmgw.v1.Host - (*Storage)(nil), // 8: llmgw.v1.Storage - (*RunningShard)(nil), // 9: llmgw.v1.RunningShard - (*GPU)(nil), // 10: llmgw.v1.GPU - (*MIGDevice)(nil), // 11: llmgw.v1.MIGDevice - (*FabricAttachment)(nil), // 12: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 13: llmgw.v1.LocalWeights - (*NodeLoad)(nil), // 14: llmgw.v1.NodeLoad - (*GPUSensor)(nil), // 15: llmgw.v1.GPUSensor - (*ShardLoad)(nil), // 16: llmgw.v1.ShardLoad - (*IBPort)(nil), // 17: llmgw.v1.IBPort - (*SubnetManager)(nil), // 18: llmgw.v1.SubnetManager - (*IMEX)(nil), // 19: llmgw.v1.IMEX - (*GPUFabricState)(nil), // 20: llmgw.v1.GPUFabricState - (*Heartbeat)(nil), // 21: llmgw.v1.Heartbeat - (*Provision)(nil), // 22: llmgw.v1.Provision - (*EngineWant)(nil), // 23: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 24: llmgw.v1.ProvisionStatus - (*Compute)(nil), // 25: llmgw.v1.Compute - (*ShardStatus)(nil), // 26: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 27: llmgw.v1.FetchProgress - (*StartShard)(nil), // 28: llmgw.v1.StartShard - (*EngineProfile)(nil), // 29: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 30: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 31: llmgw.v1.StopShard - (*DrainShard)(nil), // 32: llmgw.v1.DrainShard - (*Ack)(nil), // 33: llmgw.v1.Ack - (*LogsRequest)(nil), // 34: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 35: llmgw.v1.LogsReply - (*FabricScanRequest)(nil), // 36: llmgw.v1.FabricScanRequest - (*FabricScanReply)(nil), // 37: llmgw.v1.FabricScanReply - (*FabricTestRequest)(nil), // 38: llmgw.v1.FabricTestRequest - (*FabricTestReply)(nil), // 39: llmgw.v1.FabricTestReply - (*UpgradeAgent)(nil), // 40: llmgw.v1.UpgradeAgent - (*UpgradeReply)(nil), // 41: llmgw.v1.UpgradeReply - (*LoraAdapter)(nil), // 42: llmgw.v1.LoraAdapter - nil, // 43: llmgw.v1.ControlMessage.TraceEntry - nil, // 44: llmgw.v1.Register.LabelsEntry - nil, // 45: llmgw.v1.Register.EngineVersionsEntry - nil, // 46: llmgw.v1.GPU.NicPathsEntry - nil, // 47: llmgw.v1.IBPort.CountersEntry - nil, // 48: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 49: llmgw.v1.EngineWant.OfflineSha256Entry - nil, // 50: llmgw.v1.EngineProfile.EnvEntry + (*RestartAgent)(nil), // 4: llmgw.v1.RestartAgent + (*Firewall)(nil), // 5: llmgw.v1.Firewall + (*ControlPlanes)(nil), // 6: llmgw.v1.ControlPlanes + (*Register)(nil), // 7: llmgw.v1.Register + (*Host)(nil), // 8: llmgw.v1.Host + (*Storage)(nil), // 9: llmgw.v1.Storage + (*RunningShard)(nil), // 10: llmgw.v1.RunningShard + (*GPU)(nil), // 11: llmgw.v1.GPU + (*MIGDevice)(nil), // 12: llmgw.v1.MIGDevice + (*FabricAttachment)(nil), // 13: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 14: llmgw.v1.LocalWeights + (*NodeLoad)(nil), // 15: llmgw.v1.NodeLoad + (*GPUSensor)(nil), // 16: llmgw.v1.GPUSensor + (*ShardLoad)(nil), // 17: llmgw.v1.ShardLoad + (*IBPort)(nil), // 18: llmgw.v1.IBPort + (*SubnetManager)(nil), // 19: llmgw.v1.SubnetManager + (*IMEX)(nil), // 20: llmgw.v1.IMEX + (*GPUFabricState)(nil), // 21: llmgw.v1.GPUFabricState + (*Heartbeat)(nil), // 22: llmgw.v1.Heartbeat + (*Provision)(nil), // 23: llmgw.v1.Provision + (*EngineWant)(nil), // 24: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 25: llmgw.v1.ProvisionStatus + (*Compute)(nil), // 26: llmgw.v1.Compute + (*ShardStatus)(nil), // 27: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 28: llmgw.v1.FetchProgress + (*StartShard)(nil), // 29: llmgw.v1.StartShard + (*EngineProfile)(nil), // 30: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 31: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 32: llmgw.v1.StopShard + (*DrainShard)(nil), // 33: llmgw.v1.DrainShard + (*Ack)(nil), // 34: llmgw.v1.Ack + (*LogsRequest)(nil), // 35: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 36: llmgw.v1.LogsReply + (*FabricScanRequest)(nil), // 37: llmgw.v1.FabricScanRequest + (*FabricScanReply)(nil), // 38: llmgw.v1.FabricScanReply + (*FabricTestRequest)(nil), // 39: llmgw.v1.FabricTestRequest + (*FabricTestReply)(nil), // 40: llmgw.v1.FabricTestReply + (*UpgradeAgent)(nil), // 41: llmgw.v1.UpgradeAgent + (*UpgradeReply)(nil), // 42: llmgw.v1.UpgradeReply + (*LoraAdapter)(nil), // 43: llmgw.v1.LoraAdapter + nil, // 44: llmgw.v1.ControlMessage.TraceEntry + nil, // 45: llmgw.v1.Register.LabelsEntry + nil, // 46: llmgw.v1.Register.EngineVersionsEntry + nil, // 47: llmgw.v1.GPU.NicPathsEntry + nil, // 48: llmgw.v1.IBPort.CountersEntry + nil, // 49: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 50: llmgw.v1.EngineWant.OfflineSha256Entry + nil, // 51: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ - 6, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 21, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 26, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 35, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 37, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply - 39, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply - 41, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply + 7, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register + 22, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 27, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 36, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 38, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply + 40, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply + 42, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply 2, // 7: llmgw.v1.AgentMessage.kv_events:type_name -> llmgw.v1.KVEvents - 28, // 8: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 31, // 9: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 32, // 10: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 33, // 11: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 30, // 12: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 22, // 13: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 34, // 14: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 36, // 15: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest - 38, // 16: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest - 40, // 17: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent - 5, // 18: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes - 4, // 19: llmgw.v1.ControlMessage.firewall:type_name -> llmgw.v1.Firewall - 43, // 20: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry - 44, // 21: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 10, // 22: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 12, // 23: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 45, // 24: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 9, // 25: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 13, // 26: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 8, // 27: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 7, // 28: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 19, // 29: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX - 0, // 30: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 25, // 31: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute - 46, // 32: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 11, // 33: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice - 16, // 34: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 15, // 35: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 47, // 36: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 18, // 37: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 8, // 38: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 13, // 39: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 48, // 40: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 24, // 41: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 14, // 42: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 17, // 43: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 12, // 44: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 20, // 45: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState - 19, // 46: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX - 23, // 47: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 49, // 48: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry - 0, // 49: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 27, // 50: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 25, // 51: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute - 29, // 52: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 50, // 53: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 42, // 54: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter - 1, // 55: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 3, // 56: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 56, // [56:57] is the sub-list for method output_type - 55, // [55:56] is the sub-list for method input_type - 55, // [55:55] is the sub-list for extension type_name - 55, // [55:55] is the sub-list for extension extendee - 0, // [0:55] is the sub-list for field type_name + 29, // 8: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 32, // 9: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 33, // 10: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 34, // 11: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 31, // 12: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 23, // 13: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 35, // 14: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 37, // 15: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 39, // 16: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest + 41, // 17: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent + 6, // 18: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes + 5, // 19: llmgw.v1.ControlMessage.firewall:type_name -> llmgw.v1.Firewall + 4, // 20: llmgw.v1.ControlMessage.restart_agent:type_name -> llmgw.v1.RestartAgent + 44, // 21: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry + 45, // 22: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 11, // 23: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 13, // 24: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 46, // 25: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 10, // 26: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 14, // 27: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 9, // 28: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 8, // 29: llmgw.v1.Register.host:type_name -> llmgw.v1.Host + 20, // 30: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX + 0, // 31: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 26, // 32: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute + 47, // 33: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 12, // 34: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice + 17, // 35: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 16, // 36: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor + 48, // 37: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 19, // 38: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 9, // 39: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 14, // 40: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 49, // 41: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 25, // 42: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 15, // 43: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 18, // 44: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 13, // 45: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 21, // 46: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState + 20, // 47: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX + 24, // 48: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 50, // 49: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry + 0, // 50: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 28, // 51: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 26, // 52: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute + 30, // 53: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 51, // 54: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 43, // 55: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter + 1, // 56: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 3, // 57: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 57, // [57:58] is the sub-list for method output_type + 56, // [56:57] is the sub-list for method input_type + 56, // [56:56] is the sub-list for extension type_name + 56, // [56:56] is the sub-list for extension extendee + 0, // [0:56] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -5181,6 +5248,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_Upgrade)(nil), (*ControlMessage_ControlPlanes)(nil), (*ControlMessage_Firewall)(nil), + (*ControlMessage_RestartAgent)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -5188,7 +5256,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 50, + NumMessages: 51, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 82a6cd9..11aece0 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -72,6 +72,7 @@ message ControlMessage { UpgradeAgent upgrade = 10; ControlPlanes control_planes = 11; Firewall firewall = 13; + RestartAgent restart_agent = 14; } // trace is the W3C trace context (traceparent, and tracestate when there // is one) of the control plane's span that sent this command, so what the @@ -81,6 +82,12 @@ message ControlMessage { map trace = 12; } +// RestartAgent has the agent start again, as llmgw restart asks of every +// node: its engines keep serving and are taken over again. +message RestartAgent { + string reason = 1; +} + // Firewall is whether llmgw's firewall is kept on the node — its nftables // table, which lets the cluster's networks alone reach the engines' ports and // the node exporter — sent when an agent registers and whenever the diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index b4d6eba..f253b42 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -17,6 +17,9 @@ } ], "tags": [ + { + "name": "agents" + }, { "name": "audit" }, @@ -100,6 +103,38 @@ } ], "paths": { + "/api/v1/agents/restart": { + "post": { + "operationId": "postAgentsRestart", + "summary": "Have every node's agent start again; their engines keep serving", + "tags": [ + "agents" + ], + "parameters": [ + { + "$ref": "#/components/parameters/Actor" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentRestartView" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/audit": { "get": { "operationId": "getAudit", @@ -3554,6 +3589,18 @@ } } }, + "AgentRestartView": { + "type": "object", + "description": "AgentRestartView names the nodes whose agents were told to restart.", + "properties": { + "nodes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "CanaryPercentRequest": { "type": "object", "description": "CanaryPercentRequest changes the share of a model's requests its canary takes.", @@ -5928,6 +5975,10 @@ }, "description": "Judges are the guardrails' judge models with something wrong." }, + "stopped": { + "type": "boolean", + "description": "Stopped says the operators have stopped the service (llmgw stop): no model runs, and requests for one are answered 503." + }, "guard_off": { "type": "boolean", "description": "GuardOff says the guardrails are turned off (guard.enabled: false): no request is checked, whatever the policies say." diff --git a/docs/install.md b/docs/install.md index e846336..5151ae5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2294,20 +2294,42 @@ A cluster set up online becomes offline the same way: `bundle import` on the mirror and control plane, then the `offline:` block in `server.yaml` and a restart. -## Stopping and starting a machine +## Stopping and starting the system + +From a control plane: ``` -sudo llmgw status # each llmgw service here, and whether it runs -sudo llmgw stop # agent, control plane, mirror — whichever this machine has -sudo llmgw start -sudo llmgw restart agent # or name one: server, agent, mirror +llmgw status # does it serve; its control planes, nodes, models, store +llmgw stop --note "maintenance" # every model taken down, every request answered 503 +llmgw start # every model back as its catalog entry asks +sudo llmgw restart # every agent and control plane started again, engines serving throughout +``` + +`llmgw stop` frees every GPU and answers requests for a model with 503 +(`system_stopped`, with Retry-After) until `llmgw start`. The control planes, +agents and mirror keep running, so the console, the admin pages and `llmgw +start` keep working, and no model's definition changes. It is the cluster's +setting `serving.stopped`, so a restart does not undo it, and it is in the +audit with its note. `--wait` waits until every replica is down, or every +model has its replicas again. + +`llmgw restart` restarts every node's agent, then the other control planes +over ssh (`--ssh-user`, the one sudo was run by), then this one — the leader +last, so it hands over to a standby already restarted. Engines keep serving +and are taken over again; nothing is placed anew. + +One machine's own services are `llmgw service`: + +``` +sudo llmgw service status +sudo llmgw service stop # agent, control plane, mirror — whichever this machine has +sudo llmgw service start +sudo llmgw service restart agent # or name one: server, agent, mirror ``` Stopping a node's agent leaves its engines serving — the agent takes them -over again when it starts — unless `llmgw stop --engines` stops them too, and -the control plane then places their replicas elsewhere. Stopping the leading -control plane hands over to a standby within seconds. The whole cluster's -state is `llmgw ctl status`. +over again when it starts — unless `llmgw service stop --engines` stops them +too, and the control plane then places their replicas elsewhere. ## Removing llmgw from a machine diff --git a/docs/reference.md b/docs/reference.md index 670e34a..2d5698f 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -290,6 +290,7 @@ rather than repeated. | `secrets` | `Secrets` | [Server](#server) | | `notify` | `notify.Config` | [Server](#server) | | `mail` | `mail.Config` | [Server](#server) | +| `serving` | `ServingSettings` | [Server](#server) | | `firewall` | `FirewallSettings` | [Server](#server) | | `firewall_networks` | `[]string` | [Server](#server) | | `backup_recipients` | `[]string` | [Server](#server) | @@ -302,6 +303,7 @@ rather than repeated. | `facility` | `Facility` | [Server](#server) | | `federation` | `Federation` | [Server](#server) | | `endpoints` | `[]Endpoint` | [Server](#server) | +| `stopped` | `bool` | [ServingSettings](#servingsettings) | | `bin` | `string` | [SimConfig](#simconfig) | | `python` | `string` | [TRTLLMConfig](#trtllmconfig) | | `store` | `string` | [Threads](#threads) | @@ -2481,6 +2483,15 @@ forgotten password. Unset, none is sent and people ask an operator. Takes a [mail.Config](#mailconfig) block. +### `serving` + +`ServingSettings` + +Serving stops and starts the whole cluster's service (llmgw stop, llmgw +start). + +Takes a [ServingSettings](#servingsettings) block. + ### `firewall` `FirewallSettings` @@ -2579,6 +2590,18 @@ Endpoints is the phase-1 static replica list. It is only consulted when no model declares a profile, which is what makes a router-only smoke test against sim-engines possible without a control plane. +## ServingSettings + +### `stopped` + +`bool` + +Stopped takes every model down and answers every request for one with +503 (system_stopped), leaving the control planes, agents and mirror +running and every model's definition as it is: llmgw stop sets it, +llmgw start clears it, and the models come back as their catalog +entries ask. In force on every control plane within seconds. + ## SimConfig ### `bin` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 631da0e..0f5cde4 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -869,6 +869,14 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha a.setControlPlanes(m.ControlPlanes) case *llmgwv1.ControlMessage_Firewall: go a.applyFirewall(m.Firewall) + case *llmgwv1.ControlMessage_RestartAgent: + go func() { + a.log.Info("restarting, as the control plane asks; the engines keep serving", + "reason", m.RestartAgent.GetReason()) + // A moment for the log line to leave before the process does. + time.Sleep(300 * time.Millisecond) + exitForUpgrade() + }() case *llmgwv1.ControlMessage_Logs: reply := a.logsReply(m.Logs) // Off the receive loop: a full send queue must not stop commands being diff --git a/internal/cli/clustersettings.go b/internal/cli/clustersettings.go index c474670..f166178 100644 --- a/internal/cli/clustersettings.go +++ b/internal/cli/clustersettings.go @@ -237,6 +237,8 @@ func (w *settingsWatch) check(ctx context.Context) { // again: what it reads at every use, or not at all. var liveSettings = []string{ "guard", + // Stopping and starting the service (llmgw stop, llmgw start). + "serving", // Sent to the nodes and this machine's firewall unit as they change. "firewall", "firewall_networks", // Read by server init, node add and backups, not by the server. diff --git a/internal/cli/root.go b/internal/cli/root.go index ff3db3b..7f08832 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -63,8 +63,10 @@ func Root() *cobra.Command { newWipeCmd(), newDocsCmd(), ) - // status, start, stop and restart: this machine's llmgw services. - root.AddCommand(newServiceCmds()...) + // status, start, stop and restart: the whole system; service: this + // machine's services. + root.AddCommand(newSystemCmds()...) + root.AddCommand(newServiceCmd()) return root } diff --git a/internal/cli/server.go b/internal/cli/server.go index 60acf03..80259a7 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -186,6 +186,7 @@ func newServerCmd() *cobra.Command { rt.Start(ctx) defer rt.Stop() + rt.SetStopped(cfg.Serving.Stopped) // Which cluster this is, to federated peers: the id in the hop // header that stops a request going round two clusters that @@ -252,6 +253,8 @@ func newServerCmd() *cobra.Command { } defer ctl.Close() agentSvc = svc + // Stopped by the operators (llmgw stop): nothing placed. + ctl.Reconciler().SetStopped(cfg.Serving.Stopped) // Whether nodes keep llmgw's firewall, as the settings say now. svc.SetFirewall(func() *llmgwv1.Firewall { c := current.Load() @@ -367,6 +370,9 @@ func newServerCmd() *cobra.Command { if st := rt.Store(); st != nil { // Removing a node revokes its certificate. api.SetRevoker(st.RevokeNode) + if agentSvc != nil { + api.SetAgentRestarter(agentSvc.RestartAgents) + } api.SetAudit(st) // The cluster's settings, each change checked against this // machine's own keys before it is kept. @@ -676,6 +682,14 @@ func newServerCmd() *cobra.Command { return false } current.Store(&next) + if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "serving") }) { + rt.SetStopped(next.Serving.Stopped) + if admin != nil { + admin.Reconciler().SetStopped(next.Serving.Stopped) + } + log.Warn("the service is " + map[bool]string{true: "stopped", false: "started"}[next.Serving.Stopped] + + " by the cluster's settings") + } if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) { if err := writeFirewallState(next); err != nil { log.Warn("could not record the firewall's settings for this machine", "err", err) diff --git a/internal/cli/service.go b/internal/cli/service.go index 6f54ee1..c91e9a4 100644 --- a/internal/cli/service.go +++ b/internal/cli/service.go @@ -78,13 +78,26 @@ func hasUnit(list []string, s string) bool { return false } -func newServiceCmds() []*cobra.Command { +// newServiceCmd is llmgw service: this machine's llmgw services, one by one. +// The whole system is llmgw status, start, stop and restart. +func newServiceCmd() *cobra.Command { + c := &cobra.Command{ + Use: "service", + Short: "This machine's llmgw services: status, start, stop, restart", + Long: "The control plane (server), node agent (agent) and model mirror (mirror) on\n" + + "this machine alone, by systemd. The whole system is 'llmgw status', 'llmgw\n" + + "stop', 'llmgw start' and 'llmgw restart', run on a control plane.", + } + c.AddCommand(serviceVerbs()...) + return c +} + +func serviceVerbs() []*cobra.Command { status := &cobra.Command{ Use: "status [server|agent|mirror]...", Short: "Whether llmgw's services on this machine run", Long: "Each llmgw service this machine has — the control plane (server), the\n" + - "node agent (agent), the model mirror (mirror) — and whether it runs.\n" + - "The cluster as a whole is 'llmgw ctl status'.", + "node agent (agent), the model mirror (mirror) — and whether it runs.", RunE: func(cmd *cobra.Command, args []string) error { units, err := localUnits(cmd, args) if err != nil { diff --git a/internal/cli/service_test.go b/internal/cli/service_test.go index 9593040..949e617 100644 --- a/internal/cli/service_test.go +++ b/internal/cli/service_test.go @@ -3,11 +3,18 @@ package cli import ( "bytes" "context" + "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "runtime" "strings" "testing" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) { @@ -42,7 +49,7 @@ func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) { return } var out bytes.Buffer - c := newServiceCmds()[0] + c := serviceVerbs()[0] c.SetOut(&out) c.SetContext(context.Background()) if err := c.RunE(c, nil); err != nil { @@ -52,3 +59,46 @@ func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) { t.Errorf("%s", out.String()) } } + +func TestStopSetsTheClustersServingStopped(t *testing.T) { + var put adminapi.SettingsRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/v1/summary": + _ = json.NewEncoder(w).Encode(adminapi.Summary{}) + case r.Method == "GET" && r.URL.Path == "/api/v1/settings": + _ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 3, Body: "router:\n retries: 2\n"}) + case r.Method == "PUT" && r.URL.Path == "/api/v1/settings": + _ = json.NewDecoder(r.Body).Decode(&put) + _ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 4, Body: put.Body}) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + dir := t.TempDir() + cfg := filepath.Join(dir, "server.yaml") + _ = os.WriteFile(filepath.Join(dir, "token"), []byte("t"), 0o600) + _ = os.WriteFile(cfg, []byte("listen: \":8080\"\nagent_insecure: true\nadmin_listen: "+ + strings.TrimPrefix(srv.URL, "http://")+"\nadmin_remote: true\nadmin_token_file: "+filepath.Join(dir, "token")+"\n"), 0o600) + + var stop *cobra.Command + for _, c := range newSystemCmds() { + if c.Name() == "stop" { + stop = c + } + } + var out bytes.Buffer + stop.SetOut(&out) + stop.SetArgs([]string{"--config", cfg, "--note", "maintenance"}) + if err := stop.ExecuteContext(context.Background()); err != nil { + t.Fatalf("%v: %s", err, out.String()) + } + if put.Base != 3 || !strings.Contains(put.Body, "stopped: true") || !strings.Contains(put.Body, "retries: 2") || + put.Note != "maintenance" { + t.Errorf("sent %+v", put) + } + if !strings.Contains(out.String(), "stopping") { + t.Errorf("%s", out.String()) + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go index e4b259d..3260bee 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -153,6 +153,9 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { fmt.Fprintf(out, "STORE %s\n", strings.Join(parts, ", ")) } + if s.Stopped { + fmt.Fprintln(out, "SERVICE STOPPED: no model runs and requests are answered 503 (llmgw start)") + } if s.GuardOff { fmt.Fprintln(out, "GUARD off: no request is checked, whatever the policies say (llmgw ctl guard on)") } diff --git a/internal/cli/system.go b/internal/cli/system.go new file mode 100644 index 0000000..091649e --- /dev/null +++ b/internal/cli/system.go @@ -0,0 +1,216 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" +) + +// systemClient is the admin API of the control plane on this machine, as its +// server.yaml says, with its token. +func systemClient(configPath string) func() *ctlClient { + return func() *ctlClient { + addr := "127.0.0.1:9090" + var tokens []string + if cfg, ok := loadServerConfig(configPath); ok { + if cfg.AdminListen != "" { + addr = cfg.AdminListen + if strings.HasPrefix(addr, ":") { + addr = "127.0.0.1" + addr + } + } + tokens = append(tokens, adminTokenPath(configPath, cfg)) + } + return newCtlClient(addr, tokens...) + } +} + +// newSystemCmds are llmgw status, stop, start and restart: the whole system, +// from a control plane. +func newSystemCmds() []*cobra.Command { + var configPath string + client := systemClient("/etc/llmgw/server.yaml") + bind := func(c *cobra.Command) *cobra.Command { + c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "this control plane's configuration") + c.PreRun = func(*cobra.Command, []string) { client = systemClient(configPath) } + return c + } + call := func() *ctlClient { return client() } + + status := bind(&cobra.Command{ + Use: "status", + Short: "The whole system: whether it serves, its control planes, nodes, models and store", + Long: "Run on a control plane. Says whether the service is stopped (llmgw stop),\n" + + "and everything 'llmgw ctl status' does. One machine's services are\n" + + "'llmgw service status'.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var s adminapi.Summary + if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil { + return notAControlPlane(err) + } + printSummary(cmd, s) + return nil + }, + }) + + switchService := func(stop bool) *cobra.Command { + use, short, long := "start", "Start the whole system serving again, after llmgw stop", + "Every model comes back as its catalog entry asks: placed, loaded and\n"+ + "serving again, on every control plane within seconds. Run on a control\n"+ + "plane. The services themselves are never stopped by llmgw stop; a\n"+ + "machine's own are started with 'llmgw service start'." + if stop { + use, short, long = "stop", "Stop the whole system serving: every model taken down, requests answered 503", + "Every model's replicas are taken down, freeing every GPU, and every\n"+ + "request for a model is answered 503 (system_stopped) until 'llmgw start'.\n"+ + "The control planes, agents and mirror keep running, so the console, the\n"+ + "admin pages and llmgw start still work, and no model's definition is\n"+ + "changed. In force on every control plane within seconds; kept in the\n"+ + "cluster's settings (serving.stopped), so a restart does not undo it." + } + var note string + var wait bool + c := bind(&cobra.Command{ + Use: use, + Short: short, + Long: long, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if note == "" { + note = "llmgw " + use + } + var s adminapi.Summary + if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil { + return notAControlPlane(err) + } + if s.Stopped == stop { + fmt.Fprintf(cmd.OutOrStdout(), "the system is already %s\n", map[bool]string{true: "stopped", false: "serving"}[stop]) + return nil + } + if err := changeSettings(cmd, call, note, func(doc *settings.Doc) error { + return doc.SetYAML([]string{"serving", "stopped"}, fmt.Sprint(stop)) + }); err != nil { + return err + } + out := cmd.OutOrStdout() + if !wait { + if stop { + fmt.Fprintln(out, "stopping: every model's replicas are being taken down; llmgw status follows it") + } else { + fmt.Fprintln(out, "starting: every model is placed again as it asks; llmgw status follows it") + } + return nil + } + return waitForService(cmd, call, stop) + }, + }) + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version and in the audit") + c.Flags().BoolVar(&wait, "wait", false, "wait until every replica is down (stop) or the models have theirs (start)") + return c + } + + var sshUser string + restart := bind(&cobra.Command{ + Use: "restart", + Short: "Restart every control plane and node agent; the engines keep serving", + Long: "Every node's agent starts again, then the other control planes over ssh\n" + + "as --ssh-user, then this one — the leader last, so it hands over to a\n" + + "standby already restarted. The engines keep serving throughout and are\n" + + "taken over again; nothing is placed anew. Run on a control plane, with\n" + + "sudo. One machine's services are 'llmgw service restart'.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if os.Geteuid() != 0 { + return errors.New("restarting this control plane needs root: run it with sudo") + } + ctx, out := cmd.Context(), cmd.OutOrStdout() + var s adminapi.Summary + if err := call().get(ctx, "/api/v1/summary", &s); err != nil { + return notAControlPlane(err) + } + var agents adminapi.AgentRestartView + if err := call().post(ctx, "/api/v1/agents/restart", []byte("{}"), &agents); err != nil { + return fmt.Errorf("restarting the agents: %w", err) + } + fmt.Fprintf(out, "agents %d told to restart: %s\n", len(agents.Nodes), strings.Join(agents.Nodes, ", ")) + + // The other control planes, standbys before a leader. + var others []adminapi.ControlPlane + for _, cp := range s.ControlPlanes.Members { + if !cp.This { + others = append(others, cp) + } + } + for pass := 0; pass < 2; pass++ { + for _, cp := range others { + leading := cp.State == "leading" + if leading != (pass == 1) { + continue + } + r := &onboard.SSH{Addr: cp.Address, User: sshUser} + if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil { + return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+ + "run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res))) + } + fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State) + time.Sleep(5 * time.Second) + } + } + if _, err := os.Stat(unitDir + "/llmgw-server.service"); err == nil { + fmt.Fprintln(out, "this restarting") + if b, err := exec.CommandContext(ctx, "systemctl", "restart", "llmgw-server").CombinedOutput(); err != nil { //nolint:gosec // fixed command + return fmt.Errorf("restarting this control plane: %v: %s", err, strings.TrimSpace(string(b))) + } + fmt.Fprintln(out, "this restarted") + } + fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.") + return nil + }, + }) + restart.Flags().StringVar(&sshUser, "ssh-user", os.Getenv("SUDO_USER"), "who to log in to the other control planes as") + + return []*cobra.Command{status, switchService(true), switchService(false), restart} +} + +// waitForService waits until no replica runs (stopped) or every model has +// the replicas it asks for (started). +func waitForService(cmd *cobra.Command, call func() *ctlClient, stop bool) error { + deadline := time.Now().Add(15 * time.Minute) + for time.Now().Before(deadline) { + var s adminapi.Summary + if err := call().get(cmd.Context(), "/api/v1/summary", &s); err == nil { + if stop && s.Replicas.Ready+s.Replicas.Starting+s.Replicas.Draining == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "stopped: no replica runs") + return nil + } + if !stop && s.Models.Satisfied >= s.Models.Total { + fmt.Fprintf(cmd.OutOrStdout(), "serving: %d of %d models have the replicas they ask for\n", + s.Models.Satisfied, s.Models.Total) + return nil + } + } + select { + case <-cmd.Context().Done(): + return cmd.Context().Err() + case <-time.After(5 * time.Second): + } + } + return errors.New("not there within fifteen minutes; llmgw status says where it stands") +} + +// notAControlPlane is a failure to reach the admin API, said as what it +// most often means. +func notAControlPlane(err error) error { + return fmt.Errorf("%w\n(the whole system is stopped and started from a control plane; this machine's own "+ + "services are 'llmgw service status|start|stop|restart')", err) +} diff --git a/internal/config/config.go b/internal/config/config.go index 8ae1c60..7c9486b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -151,6 +151,10 @@ type Server struct { // forgotten password. Unset, none is sent and people ask an operator. Mail mail.Config `koanf:"mail"` + // Serving stops and starts the whole cluster's service (llmgw stop, llmgw + // start). + Serving ServingSettings `koanf:"serving"` + // Firewall turns llmgw's firewall on every machine on or off. Firewall FirewallSettings `koanf:"firewall"` diff --git a/internal/config/serving.go b/internal/config/serving.go new file mode 100644 index 0000000..d624c4f --- /dev/null +++ b/internal/config/serving.go @@ -0,0 +1,11 @@ +package config + +// ServingSettings is whether the cluster serves at all. +type ServingSettings struct { + // Stopped takes every model down and answers every request for one with + // 503 (system_stopped), leaving the control planes, agents and mirror + // running and every model's definition as it is: llmgw stop sets it, + // llmgw start clears it, and the models come back as their catalog + // entries ask. In force on every control plane within seconds. + Stopped bool `koanf:"stopped"` +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index ac21726..a574db9 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -70,6 +70,8 @@ type API struct { // guard is the guardrail policies' library and attachments. guard *GuardAdmin settings *Settings + // restartAgents tells every connected agent to start again. + restartAgents func(reason string) []string // revoke records a removed node, so its certificate is refused. revoke func(ctx context.Context, nodeID string, at time.Time) error @@ -114,6 +116,7 @@ func (a *API) Register(mux Mux) { a.registerQuotas(mux) a.registerGuard(mux) a.registerSettings(mux) + mux.HandleFunc("POST /api/v1/agents/restart", a.restartAgentsHandler) mux.HandleFunc("GET /api/v1/audit", a.auditEvents) mux.HandleFunc("GET /api/v1/usage/export", a.usageExport) mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest) diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index 933d0ab..e2aca76 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -263,6 +263,9 @@ var Routes = []Route{ {Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy", Request: GuardAttachmentRequest{}, Response: guard.Attachment{}}, + {Method: "POST", Path: "/api/v1/agents/restart", Summary: "Have every node's agent start again; their engines keep serving", + Response: AgentRestartView{}}, + // The cluster's settings: everything in server.yaml that is not one // machine's own, the same on every control plane. {Method: "GET", Path: "/api/v1/settings", Summary: "The cluster's settings in force, or one earlier version", diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index 92baf57..0a31539 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -44,6 +44,9 @@ type Summary struct { ControlPlanes ControlPlanes `json:"control_planes,omitzero"` // Judges are the guardrails' judge models with something wrong. Judges []JudgeView `json:"judges,omitempty"` + // Stopped says the operators have stopped the service (llmgw stop): no + // model runs, and requests for one are answered 503. + Stopped bool `json:"stopped,omitempty"` // GuardOff says the guardrails are turned off (guard.enabled: false): // no request is checked, whatever the policies say. GuardOff bool `json:"guard_off,omitempty"` @@ -153,7 +156,7 @@ const idleThreshold = 10 * time.Minute // Summary builds the fleet summary. func (a *API) Summary() Summary { - s := Summary{Version: version.Version} + s := Summary{Version: version.Version, Stopped: a.ctl.Reconciler().Stopped()} if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() { s.GuardOff = true } diff --git a/internal/control/adminapi/system.go b/internal/control/adminapi/system.go new file mode 100644 index 0000000..de20cdd --- /dev/null +++ b/internal/control/adminapi/system.go @@ -0,0 +1,22 @@ +package adminapi + +import ( + "net/http" +) + +// AgentRestartView names the nodes whose agents were told to restart. +type AgentRestartView struct { + Nodes []string `json:"nodes"` +} + +// SetAgentRestarter enables POST /api/v1/agents/restart. +func (a *API) SetAgentRestarter(f func(reason string) []string) { a.restartAgents = f } + +func (a *API) restartAgentsHandler(w http.ResponseWriter, r *http.Request) { + if a.restartAgents == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no agents to restart") + return + } + nodes := a.restartAgents("llmgw restart, by " + actorOf(r)) + writeJSON(w, http.StatusAccepted, AgentRestartView{Nodes: nodes}) +} diff --git a/internal/control/agentsvc/firewall.go b/internal/control/agentsvc/firewall.go index 7d10377..4d48d2f 100644 --- a/internal/control/agentsvc/firewall.go +++ b/internal/control/agentsvc/firewall.go @@ -1,6 +1,8 @@ package agentsvc import ( + "time" + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" ) @@ -40,3 +42,20 @@ func (s *Service) BroadcastFirewall() { } } } + +// RestartAgents has every connected agent start again, one at a time a +// moment apart, and names those it told. Their engines keep serving. +func (s *Service) RestartAgents(reason string) []string { + msg := &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_RestartAgent{ + RestartAgent: &llmgwv1.RestartAgent{Reason: reason}}} + var told []string + for _, node := range s.ConnectedNodes() { + if err := s.send(node, msg); err != nil { + s.log.Warn("could not tell a node to restart its agent", "node", node, "err", err) + continue + } + told = append(told, node) + time.Sleep(time.Second) + } + return told +} diff --git a/internal/control/reconciler/demand.go b/internal/control/reconciler/demand.go index eb58c20..a656283 100644 --- a/internal/control/reconciler/demand.go +++ b/internal/control/reconciler/demand.go @@ -81,6 +81,10 @@ func (r *Reconciler) Want(model string) bool { // feature: a cluster with more models than GPUs can hold every model in its // catalog and only the wanted ones in its cards. func (r *Reconciler) desiredFor(m catalog.Model) int { + // Stopped by the operators: nothing runs, whatever the catalog asks. + if r.stopped.Load() { + return 0 + } base := r.baseFor(m) if base == 0 { return 0 diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 9e94c43..13d9c25 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -307,6 +307,9 @@ type Reconciler struct { startedAt time.Time lastNodeCount int steadySince time.Time + // stopped is the operators having stopped the service: every model is + // wanted at no replicas until they start it again. + stopped atomic.Bool // stoodBy is whether the last pass found this process a standby: one // that then takes over settles again, as at start. stoodBy bool @@ -484,6 +487,17 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) { } } +// SetStopped stops every model, or lets each run as its catalog entry +// asks again, from the next pass. +func (r *Reconciler) SetStopped(stopped bool) { + if r.stopped.Swap(stopped) != stopped { + r.Notify() + } +} + +// Stopped is whether the operators have stopped the service. +func (r *Reconciler) Stopped() bool { return r.stopped.Load() } + // resettle opens the settling window again from now, for a control plane // that has just become the leader. func (r *Reconciler) resettle() { diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index bcd1190..2a8b466 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -2133,3 +2133,27 @@ func TestAnAdoptedShardKeepsWhatItComputesOn(t *testing.T) { t.Fatalf("%+v", reps) } } + +func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.r.Reconcile(t.Context()) + h.markReady() + if len(h.r.ReadyReplicas()) != 1 { + t.Fatal("nothing ready to stop") + } + h.r.SetStopped(true) + h.r.Reconcile(t.Context()) + for _, rep := range h.r.Replicas() { + if rep.State == Ready { + t.Fatalf("stopped, a replica still serves: %+v", rep) + } + } + h.r.SetStopped(false) + for range 3 { + h.r.Reconcile(t.Context()) + h.markReady() + } + if len(h.r.ReadyReplicas()) != 1 { + t.Errorf("started again, %d ready", len(h.r.ReadyReplicas())) + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 51eb1c2..8cb6e52 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -368,6 +368,7 @@ var Reference = []Block{ {Key: "secrets", Type: "Secrets", Doc: "Secrets is where secrets kept outside files are read from, and the key\nthe store's own secrets are sealed under.", Nested: "config.Secrets"}, {Key: "notify", Type: "notify.Config", Doc: "Notify sends what needs a person — a model parked, a node lost, a\nbudget spent, new weights published — to webhooks, signed, without a\nPrometheus and Alertmanager in between. Empty sends nothing.", Nested: "notify.Config"}, {Key: "mail", Type: "mail.Config", Doc: "Mail is where the gateway's few emails go out from: a link to set a\nforgotten password. Unset, none is sent and people ask an operator.", Nested: "mail.Config"}, + {Key: "serving", Type: "ServingSettings", Doc: "Serving stops and starts the whole cluster's service (llmgw stop, llmgw\nstart).", Nested: "config.ServingSettings"}, {Key: "firewall", Type: "FirewallSettings", Doc: "Firewall turns llmgw's firewall on every machine on or off.", Nested: "config.FirewallSettings"}, {Key: "firewall_networks", Type: "[]string", Doc: "FirewallNetworks are networks, besides those each machine reaches the\ncluster on, that llmgw's internal ports answer: a monitoring host on\nanother subnet, nodes behind a router. Written into every machine's\nfirewall by `server init`, `node add` and `server add`.", Nested: ""}, {Key: "backup_recipients", Type: "[]string", Doc: "BackupRecipients encrypt every archive `llmgw server backup` writes, to\nwhoever holds one of their keys: age recipients (age1…) or SSH public\nkeys (ssh-ed25519 …, ssh-rsa …), as `age` takes them. An archive holds\nthe certificate authority's key and every secret in the store, and is\ncopied off the machine; encrypted, the copy is safe wherever it lands.\nEmpty writes archives unencrypted, readable by root only.", Nested: ""}, @@ -381,6 +382,9 @@ var Reference = []Block{ {Key: "federation", Type: "Federation", Doc: "Federation is how this cluster takes part in federation with other\nllmgw clusters. The peers themselves are added at runtime, with\n`llmgw ctl federation add`, and kept in the store.", Nested: "config.Federation"}, {Key: "endpoints", Type: "[]Endpoint", Doc: "Endpoints is the phase-1 static replica list. It is only consulted when\nno model declares a profile, which is what makes a router-only smoke test\nagainst sim-engines possible without a control plane.", Nested: ""}, }}, + {Package: "config", Name: "ServingSettings", Title: "ServingSettings", Doc: "", Fields: []Field{ + {Key: "stopped", Type: "bool", Doc: "Stopped takes every model down and answers every request for one with\n503 (system_stopped), leaving the control planes, agents and mirror\nrunning and every model's definition as it is: llmgw stop sets it,\nllmgw start clears it, and the models come back as their catalog\nentries ask. In force on every control plane within seconds.", Nested: ""}, + }}, {Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{ {Key: "bin", Type: "string", Doc: "Bin is the llmgw binary itself; the engine is one of its subcommands.\nEmpty means whatever started this agent, which is right almost always.", Nested: ""}, }}, diff --git a/internal/docgen/render.go b/internal/docgen/render.go index 85108d4..540feed 100644 --- a/internal/docgen/render.go +++ b/internal/docgen/render.go @@ -31,6 +31,7 @@ var Sources = []string{ "internal/guard/judge.go", "internal/config/oidc.go", "internal/config/firewall.go", + "internal/config/serving.go", "internal/mail/mail.go", "internal/notify/notify.go", "internal/secrets/secrets.go", diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 8867744..afbf243 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -310,7 +310,7 @@ func (r *Router) Handler() http.Handler { } if r.auth == nil { - return r.refuseLoops(mux) + return r.refuseWhileStopped(r.refuseLoops(mux)) } exempt := []string{"/health", "/ready", "/leader", "/", "/about", "/_static/", "/_lang/", "/claim/", "/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/", @@ -324,7 +324,22 @@ func (r *Router) Handler() http.Handler { // Wrapping the whole mux rather than checking inside each handler: adding a // route must not be able to add an unauthenticated one by omission. Health // stays open because a load balancer has no key. - return r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux)) + return r.refuseWhileStopped(r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux))) +} + +// refuseWhileStopped answers every request for a model with 503 while the +// operators have stopped the service (llmgw stop): the models are being taken +// down, and a client is better told so plainly than left to time out. +func (r *Router) refuseWhileStopped(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if r.stopped.Load() && req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/v1/") { + w.Header().Set("Retry-After", "300") + writeErrorCode(w, http.StatusServiceUnavailable, "server_error", "system_stopped", + "this gateway's service has been stopped by its operators; it answers again once they start it") + return + } + next.ServeHTTP(w, req) + }) } // parsedRequest is what the router needs from a request. Everything else in the diff --git a/internal/router/router.go b/internal/router/router.go index eb98726..08fbc2e 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -115,7 +115,9 @@ type Router struct { oidc config.OIDC // mail sends what the portal mails: a password reset link. mail mail.Config - scim config.SCIM + // stopped is the operators having stopped the service (llmgw stop). + stopped atomic.Bool + scim config.SCIM // threads holds conversations and memory notes. It is store when auth is on, // its own file when only the threads block is configured, and nil when @@ -192,6 +194,9 @@ func (r *Router) SetGuard(g guard.Guard) { r.guard = g } // SetGuardLibrary is the policies the guard was built from, for the pages. func (r *Router) SetGuardLibrary(c func() guard.Config) { r.guardLibrary = c } +// SetStopped refuses, or again serves, every request for a model. +func (r *Router) SetStopped(stopped bool) { r.stopped.Store(stopped) } + // SetRegistry gives the mirror page somewhere to look models up. func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } diff --git a/internal/router/router_test.go b/internal/router/router_test.go index bad3f04..9103f2f 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -748,3 +748,24 @@ func TestModelsEndpointListsOnlyWhatTheKeyMayUse(t *testing.T) { t.Errorf("listed %v; the key may use real and its alias only", ids) } } + +func TestAStoppedServiceAnswers503AndStillListsModels(t *testing.T) { + e := newFakeEngine(t, "m") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})) + h := rt.Handler() + rt.SetStopped(true) + rec := post(t, h, map[string]any{"model": "m", "messages": []map[string]string{{"role": "user", "content": "hi"}}}, nil) + if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "system_stopped") || + rec.Header().Get("Retry-After") == "" { + t.Fatalf("stopped: %d %s", rec.Code, rec.Body.String()) + } + list := httptest.NewRecorder() + h.ServeHTTP(list, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + if list.Code == http.StatusServiceUnavailable { + t.Error("listing models was refused too") + } + rt.SetStopped(false) + if rec := post(t, h, map[string]any{"model": "m", "messages": []map[string]string{{"role": "user", "content": "hi"}}}, nil); rec.Code == http.StatusServiceUnavailable { + t.Errorf("started again, still refused: %s", rec.Body.String()) + } +}