From daed835d88505f476547c21bafdc51f05c521915 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 28 Sep 2026 23:21:23 +0200 Subject: [PATCH] Measure each node's CPU and memory, and give every node a page The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 393 +++++++++++++----- api/proto/llmgw/v1/agent.proto | 35 ++ internal/agent/agent.go | 33 ++ internal/agent/discovery/load.go | 260 ++++++++++++ internal/agent/discovery/load_test.go | 96 +++++ internal/cli/server.go | 1 + internal/control/adminapi/api.go | 53 +++ internal/control/agentsvc/heartbeat_test.go | 22 + internal/control/agentsvc/service.go | 21 +- internal/control/fleetmetrics/fleetmetrics.go | 60 +++ .../control/fleetmetrics/fleetmetrics_test.go | 37 ++ internal/control/inventory/inventory.go | 25 ++ internal/router/handlers.go | 2 + internal/web/node_page.go | 167 ++++++++ internal/web/node_page_test.go | 86 ++++ internal/web/static/base.css | 8 + internal/web/templates/dashboard.html | 18 +- internal/web/templates/node.html | 69 +++ internal/web/web.go | 5 + 19 files changed, 1289 insertions(+), 102 deletions(-) create mode 100644 internal/agent/discovery/load.go create mode 100644 internal/agent/discovery/load_test.go create mode 100644 internal/web/node_page.go create mode 100644 internal/web/node_page_test.go create mode 100644 internal/web/templates/node.html diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index abe6661d..5265e854 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15, 0} } // AgentMessage is anything a node tells the control plane. @@ -1202,6 +1202,179 @@ func (x *LocalWeights) GetShared() bool { return false } +// NodeLoad is a node's use of its own CPU and memory, and each engine's share. +// +// Carried on the heartbeat rather than scraped from the node: the control +// plane already exports the fleet from one place, and a node exporter per +// machine would be a scrape target, a port and a firewall rule each. +type NodeLoad struct { + state protoimpl.MessageState `protogen:"open.v1"` + // cpu_percent is the whole machine, 0 to 100 however many cores it has. + CpuPercent float64 `protobuf:"fixed64,1,opt,name=cpu_percent,json=cpuPercent,proto3" json:"cpu_percent,omitempty"` + CpuCount int32 `protobuf:"varint,2,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"` + // The kernel's run-queue averages over 1, 5 and 15 minutes. + Load1 float64 `protobuf:"fixed64,3,opt,name=load1,proto3" json:"load1,omitempty"` + Load5 float64 `protobuf:"fixed64,4,opt,name=load5,proto3" json:"load5,omitempty"` + Load15 float64 `protobuf:"fixed64,5,opt,name=load15,proto3" json:"load15,omitempty"` + // memory_used_bytes is total less available: what the machine could not + // hand to a new process without evicting something. + MemoryUsedBytes int64 `protobuf:"varint,6,opt,name=memory_used_bytes,json=memoryUsedBytes,proto3" json:"memory_used_bytes,omitempty"` + MemoryAvailableBytes int64 `protobuf:"varint,7,opt,name=memory_available_bytes,json=memoryAvailableBytes,proto3" json:"memory_available_bytes,omitempty"` + Shards []*ShardLoad `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NodeLoad) Reset() { + *x = NodeLoad{} + mi := &file_llmgw_v1_agent_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NodeLoad) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeLoad) ProtoMessage() {} + +func (x *NodeLoad) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeLoad.ProtoReflect.Descriptor instead. +func (*NodeLoad) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} +} + +func (x *NodeLoad) GetCpuPercent() float64 { + if x != nil { + return x.CpuPercent + } + return 0 +} + +func (x *NodeLoad) GetCpuCount() int32 { + if x != nil { + return x.CpuCount + } + return 0 +} + +func (x *NodeLoad) GetLoad1() float64 { + if x != nil { + return x.Load1 + } + return 0 +} + +func (x *NodeLoad) GetLoad5() float64 { + if x != nil { + return x.Load5 + } + return 0 +} + +func (x *NodeLoad) GetLoad15() float64 { + if x != nil { + return x.Load15 + } + return 0 +} + +func (x *NodeLoad) GetMemoryUsedBytes() int64 { + if x != nil { + return x.MemoryUsedBytes + } + return 0 +} + +func (x *NodeLoad) GetMemoryAvailableBytes() int64 { + if x != nil { + return x.MemoryAvailableBytes + } + return 0 +} + +func (x *NodeLoad) GetShards() []*ShardLoad { + if x != nil { + return x.Shards + } + return nil +} + +// ShardLoad is one engine process group's use. +type ShardLoad struct { + state protoimpl.MessageState `protogen:"open.v1"` + ShardId string `protobuf:"bytes,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` + // cpu_cores is in cores, the way top counts a process: 2.5 is two and a + // half cores busy, which a percentage of the machine would hide on a large + // one. + CpuCores float64 `protobuf:"fixed64,2,opt,name=cpu_cores,json=cpuCores,proto3" json:"cpu_cores,omitempty"` + MemoryRssBytes int64 `protobuf:"varint,3,opt,name=memory_rss_bytes,json=memoryRssBytes,proto3" json:"memory_rss_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ShardLoad) Reset() { + *x = ShardLoad{} + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ShardLoad) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShardLoad) ProtoMessage() {} + +func (x *ShardLoad) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead. +func (*ShardLoad) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *ShardLoad) GetShardId() string { + if x != nil { + return x.ShardId + } + return "" +} + +func (x *ShardLoad) GetCpuCores() float64 { + if x != nil { + return x.CpuCores + } + return 0 +} + +func (x *ShardLoad) GetMemoryRssBytes() int64 { + if x != nil { + return x.MemoryRssBytes + } + return 0 +} + type Heartbeat struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -1238,13 +1411,17 @@ type Heartbeat struct { // than "nothing to say", as for weights. ProvisionChanged bool `protobuf:"varint,10,opt,name=provision_changed,json=provisionChanged,proto3" json:"provision_changed,omitempty"` Provision []*ProvisionStatus `protobuf:"bytes,11,rep,name=provision,proto3" json:"provision,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // load is how busy the machine is, sampled over the interval since the + // previous heartbeat. Absent on the first one, which has nothing to take a + // difference from, and from an agent on a system without /proc. + Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1256,7 +1433,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1269,7 +1446,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *Heartbeat) GetNodeId() string { @@ -1349,6 +1526,13 @@ func (x *Heartbeat) GetProvision() []*ProvisionStatus { return nil } +func (x *Heartbeat) GetLoad() *NodeLoad { + if x != nil { + return x.Load + } + return nil +} + // Provision is the software a node should have: which engines, at which // versions. Nothing more. // @@ -1368,7 +1552,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1380,7 +1564,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1393,7 +1577,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *Provision) GetEngines() []*EngineWant { @@ -1414,7 +1598,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1426,7 +1610,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1439,7 +1623,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *EngineWant) GetEngine() string { @@ -1472,7 +1656,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1484,7 +1668,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1497,7 +1681,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *ProvisionStatus) GetEngine() string { @@ -1561,7 +1745,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1573,7 +1757,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1586,7 +1770,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *ShardStatus) GetNodeId() string { @@ -1660,7 +1844,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1672,7 +1856,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1685,7 +1869,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *FetchProgress) GetSource() string { @@ -1741,7 +1925,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1753,7 +1937,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1766,7 +1950,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *StartShard) GetReplicaId() string { @@ -1893,7 +2077,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1905,7 +2089,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1918,7 +2102,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *EngineProfile) GetName() string { @@ -2014,7 +2198,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2026,7 +2210,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2039,7 +2223,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *RemoveWeights) GetPaths() []string { @@ -2059,7 +2243,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2071,7 +2255,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2084,7 +2268,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *StopShard) GetShardId() string { @@ -2104,7 +2288,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2116,7 +2300,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2129,7 +2313,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *DrainShard) GetShardId() string { @@ -2151,7 +2335,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2163,7 +2347,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2176,7 +2360,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *Ack) GetMessageId() string { @@ -2220,7 +2404,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2232,7 +2416,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2245,7 +2429,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *LogsRequest) GetRequestId() string { @@ -2285,7 +2469,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2297,7 +2481,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2310,7 +2494,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *LogsReply) GetRequestId() string { @@ -2448,7 +2632,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" + - "\x06shared\x18\x03 \x01(\bR\x06shared\"\xc8\x04\n" + + "\x06shared\x18\x03 \x01(\bR\x06shared\"\x9b\x02\n" + + "\bNodeLoad\x12\x1f\n" + + "\vcpu_percent\x18\x01 \x01(\x01R\n" + + "cpuPercent\x12\x1b\n" + + "\tcpu_count\x18\x02 \x01(\x05R\bcpuCount\x12\x14\n" + + "\x05load1\x18\x03 \x01(\x01R\x05load1\x12\x14\n" + + "\x05load5\x18\x04 \x01(\x01R\x05load5\x12\x16\n" + + "\x06load15\x18\x05 \x01(\x01R\x06load15\x12*\n" + + "\x11memory_used_bytes\x18\x06 \x01(\x03R\x0fmemoryUsedBytes\x124\n" + + "\x16memory_available_bytes\x18\a \x01(\x03R\x14memoryAvailableBytes\x12+\n" + + "\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\"m\n" + + "\tShardLoad\x12\x19\n" + + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + + "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\xf0\x04\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -2461,7 +2659,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x12+\n" + "\x11provision_changed\x18\n" + " \x01(\bR\x10provisionChanged\x127\n" + - "\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x1aA\n" + + "\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x12&\n" + + "\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + @@ -2584,7 +2783,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 27) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -2596,62 +2795,66 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*GPU)(nil), // 7: llmgw.v1.GPU (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights - (*Heartbeat)(nil), // 10: llmgw.v1.Heartbeat - (*Provision)(nil), // 11: llmgw.v1.Provision - (*EngineWant)(nil), // 12: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 13: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 14: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 15: llmgw.v1.FetchProgress - (*StartShard)(nil), // 16: llmgw.v1.StartShard - (*EngineProfile)(nil), // 17: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 18: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 19: llmgw.v1.StopShard - (*DrainShard)(nil), // 20: llmgw.v1.DrainShard - (*Ack)(nil), // 21: llmgw.v1.Ack - (*LogsRequest)(nil), // 22: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 23: llmgw.v1.LogsReply - nil, // 24: llmgw.v1.Register.LabelsEntry - nil, // 25: llmgw.v1.Register.EngineVersionsEntry - nil, // 26: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 27: llmgw.v1.EngineProfile.EnvEntry + (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad + (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad + (*Heartbeat)(nil), // 12: llmgw.v1.Heartbeat + (*Provision)(nil), // 13: llmgw.v1.Provision + (*EngineWant)(nil), // 14: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 15: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 16: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 17: llmgw.v1.FetchProgress + (*StartShard)(nil), // 18: llmgw.v1.StartShard + (*EngineProfile)(nil), // 19: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 20: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 21: llmgw.v1.StopShard + (*DrainShard)(nil), // 22: llmgw.v1.DrainShard + (*Ack)(nil), // 23: llmgw.v1.Ack + (*LogsRequest)(nil), // 24: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 25: llmgw.v1.LogsReply + nil, // 26: llmgw.v1.Register.LabelsEntry + nil, // 27: llmgw.v1.Register.EngineVersionsEntry + nil, // 28: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 29: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 10, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 14, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 23, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 16, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 19, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 20, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 21, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 18, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 11, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 22, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 24, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 12, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 16, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 25, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 18, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 21, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 22, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 23, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 20, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 13, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 24, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 26, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 25, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 27, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host 0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 5, // 20: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 21: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 26, // 22: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 13, // 23: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 12, // 24: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 25: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 15, // 26: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 17, // 27: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 27, // 28: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 29: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 30: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 30, // [30:31] is the sub-list for method output_type - 29, // [29:30] is the sub-list for method input_type - 29, // [29:29] is the sub-list for extension type_name - 29, // [29:29] is the sub-list for extension extendee - 0, // [0:29] is the sub-list for field type_name + 11, // 20: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 5, // 21: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 22: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 28, // 23: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 15, // 24: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 25: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 14, // 26: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 27: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 17, // 28: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 19, // 29: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 29, // 30: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 31: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 32: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 32, // [32:33] is the sub-list for method output_type + 31, // [31:32] is the sub-list for method input_type + 31, // [31:31] is the sub-list for extension type_name + 31, // [31:31] is the sub-list for extension extendee + 0, // [0:31] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -2680,7 +2883,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 27, + NumMessages: 29, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index abe9771c..908a3fdd 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -231,6 +231,36 @@ message LocalWeights { bool shared = 3; } +// NodeLoad is a node's use of its own CPU and memory, and each engine's share. +// +// Carried on the heartbeat rather than scraped from the node: the control +// plane already exports the fleet from one place, and a node exporter per +// machine would be a scrape target, a port and a firewall rule each. +message NodeLoad { + // cpu_percent is the whole machine, 0 to 100 however many cores it has. + double cpu_percent = 1; + int32 cpu_count = 2; + // The kernel's run-queue averages over 1, 5 and 15 minutes. + double load1 = 3; + double load5 = 4; + double load15 = 5; + // memory_used_bytes is total less available: what the machine could not + // hand to a new process without evicting something. + int64 memory_used_bytes = 6; + int64 memory_available_bytes = 7; + repeated ShardLoad shards = 8; +} + +// ShardLoad is one engine process group's use. +message ShardLoad { + string shard_id = 1; + // cpu_cores is in cores, the way top counts a process: 2.5 is two and a + // half cores busy, which a percentage of the machine would hide on a large + // one. + double cpu_cores = 2; + int64 memory_rss_bytes = 3; +} + message Heartbeat { string node_id = 1; // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on @@ -271,6 +301,11 @@ message Heartbeat { // than "nothing to say", as for weights. bool provision_changed = 10; repeated ProvisionStatus provision = 11; + + // load is how busy the machine is, sampled over the interval since the + // previous heartbeat. Absent on the first one, which has nothing to take a + // difference from, and from an agent on a system without /proc. + NodeLoad load = 12; } // Provision is the software a node should have: which engines, at which diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 1c7fba61..f38274ed 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -352,6 +352,8 @@ type Agent struct { // failover the old stream's heartbeat loop can still be running when the // new one's starts. lastWeights string + // load measures the machine between heartbeats. + load *discovery.LoadSampler // fetchWatchers are the shards waiting on each source, so progress on one // download reaches every shard that asked for it. A fetch is single-flight @@ -396,6 +398,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er provStatus: map[string]provision.Status{}, logRing: ring, provKick: make(chan struct{}, 1), + load: discovery.NewLoadSampler(), } a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus} return a, nil @@ -883,9 +886,39 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { }) } } + hb.Load = a.sampleLoad() return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}} } +// sampleLoad measures the machine and each running engine since the last +// heartbeat. Nil on the first, which has nothing to compare with. +func (a *Agent) sampleLoad() *llmgwv1.NodeLoad { + if a.load == nil { + return nil + } + pids := map[string]int{} + for _, st := range a.sup.Statuses() { + if st.PID > 0 { + pids[st.ID] = st.PID + } + } + l, ok := a.load.Sample(pids) + if !ok { + return nil + } + out := &llmgwv1.NodeLoad{ + CpuPercent: l.CPUPercent, CpuCount: int32(l.CPUCount), + Load1: l.Load1, Load5: l.Load5, Load15: l.Load15, + MemoryUsedBytes: l.MemUsed, MemoryAvailableBytes: l.MemAvail, + } + for _, sh := range l.Shards { + out.Shards = append(out.Shards, &llmgwv1.ShardLoad{ + ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes, + }) + } + return out +} + // weightsFingerprint is enough to notice a model appearing, disappearing or // finishing its download. // diff --git a/internal/agent/discovery/load.go b/internal/agent/discovery/load.go new file mode 100644 index 00000000..c727464f --- /dev/null +++ b/internal/agent/discovery/load.go @@ -0,0 +1,260 @@ +package discovery + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// Load is how busy a node is: the machine as a whole, and each engine's share. +type Load struct { + CPUPercent float64 + CPUCount int + Load1 float64 + Load5 float64 + Load15 float64 + MemUsed int64 + MemAvail int64 + Shards []ShardLoad +} + +// ShardLoad is one engine process group's use. +type ShardLoad struct { + ID string + CPUCores float64 + RSSBytes int64 +} + +// userHZ is the unit /proc counts CPU time in. Fixed at 100 by the kernel's +// ABI on every architecture a node runs, whatever the internal tick rate. +const userHZ = 100 + +// LoadSampler measures load as the difference between two readings of /proc. +// +// CPU time is a counter, so a percentage needs two samples and the time +// between them. The sampler keeps the previous one; the first call has +// nothing to compare with and reports no load at all, rather than a zero that +// would read as an idle machine. +// +// Linux only, like every node. Elsewhere the files are missing and it reports +// nothing, as an older agent does. +type LoadSampler struct { + root string + now func() time.Time + + mu sync.Mutex + prevAt time.Time + prevCPU cpuTimes + prevPG map[int]groupUse // process group -> what it has used +} + +// groupUse is a process group's CPU time so far and its resident memory now. +type groupUse struct { + jiffies uint64 + rssPages int64 +} + +type cpuTimes struct{ total, idle uint64 } + +// NewLoadSampler reads the real /proc. +func NewLoadSampler() *LoadSampler { return newLoadSampler("/proc", time.Now) } + +func newLoadSampler(root string, now func() time.Time) *LoadSampler { + return &LoadSampler{root: root, now: now} +} + +// Sample reads the machine and the process groups led by the given shards' +// PIDs, keyed by shard ID. False when there is nothing to report yet. +func (s *LoadSampler) Sample(shards map[string]int) (Load, bool) { + cpu, ok := s.cpu() + if !ok { + return Load{}, false + } + groups := s.groups() + at := s.now() + + s.mu.Lock() + prevAt, prevCPU, prevPG := s.prevAt, s.prevCPU, s.prevPG + s.prevAt, s.prevCPU, s.prevPG = at, cpu, groups + s.mu.Unlock() + + if prevAt.IsZero() || cpu.total <= prevCPU.total { + return Load{}, false + } + l := Load{CPUCount: s.cpuCount()} + dTotal := float64(cpu.total - prevCPU.total) + dIdle := float64(cpu.idle - prevCPU.idle) + l.CPUPercent = clamp((dTotal-dIdle)/dTotal*100, 0, 100) + l.Load1, l.Load5, l.Load15 = s.loadavg() + total, avail := s.meminfo() + if total > 0 { + l.MemUsed, l.MemAvail = total-avail, avail + } + + secs := at.Sub(prevAt).Seconds() + for id, pid := range shards { + if pid <= 0 { + continue + } + now := groups[pid] + sl := ShardLoad{ID: id, RSSBytes: now.rssPages * int64(os.Getpagesize())} + if before, ok := prevPG[pid]; ok && secs > 0 && now.jiffies >= before.jiffies { + sl.CPUCores = float64(now.jiffies-before.jiffies) / userHZ / secs + } + l.Shards = append(l.Shards, sl) + } + return l, true +} + +func clamp(v, lo, hi float64) float64 { return max(lo, min(v, hi)) } + +// cpu reads the aggregate line of /proc/stat. Idle includes iowait: a core +// waiting on the disk is not doing work. +func (s *LoadSampler) cpu() (cpuTimes, bool) { + b, err := os.ReadFile(filepath.Join(s.root, "stat")) + if err != nil { + return cpuTimes{}, false + } + line, _, _ := bytes.Cut(b, []byte("\n")) + f := strings.Fields(string(line)) + if len(f) < 5 || f[0] != "cpu" { + return cpuTimes{}, false + } + var t cpuTimes + for i, v := range f[1:] { + // guest and guest_nice are already counted in user and nice. + if i >= 8 { + break + } + n, _ := strconv.ParseUint(v, 10, 64) + t.total += n + if i == 3 || i == 4 { + t.idle += n + } + } + return t, true +} + +func (s *LoadSampler) cpuCount() int { + b, err := os.ReadFile(filepath.Join(s.root, "stat")) + if err != nil { + return 0 + } + n := 0 + for _, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, "cpu") && len(line) > 3 && line[3] >= '0' && line[3] <= '9' { + n++ + } + } + return n +} + +func (s *LoadSampler) loadavg() (l1, l5, l15 float64) { + b, err := os.ReadFile(filepath.Join(s.root, "loadavg")) + if err != nil { + return 0, 0, 0 + } + f := strings.Fields(string(b)) + if len(f) < 3 { + return 0, 0, 0 + } + l1, _ = strconv.ParseFloat(f[0], 64) + l5, _ = strconv.ParseFloat(f[1], 64) + l15, _ = strconv.ParseFloat(f[2], 64) + return l1, l5, l15 +} + +// meminfo returns MemTotal and MemAvailable in bytes. +func (s *LoadSampler) meminfo() (total, avail int64) { + f, err := os.Open(filepath.Join(s.root, "meminfo")) + if err != nil { + return 0, 0 + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + k, v, ok := strings.Cut(sc.Text(), ":") + if !ok { + continue + } + fields := strings.Fields(v) + if len(fields) == 0 { + continue + } + kb, _ := strconv.ParseInt(fields[0], 10, 64) + switch k { + case "MemTotal": + total = kb * 1024 + case "MemAvailable": + avail = kb * 1024 + } + } + return total, avail +} + +// procStat is what one /proc//stat says that load needs. +type procStat struct { + pgrp int + jiffies uint64 + rssPages int64 +} + +// readProcStat parses /proc//stat. The command name is in parentheses +// and may itself hold spaces and parentheses, so fields are counted from the +// last closing one. +func readProcStat(path string) (procStat, bool) { + b, err := os.ReadFile(path) + if err != nil { + return procStat{}, false + } + i := bytes.LastIndexByte(b, ')') + if i < 0 { + return procStat{}, false + } + f := strings.Fields(string(b[i+1:])) + // After the name: state(0) ppid(1) pgrp(2) ... utime(11) stime(12) ... rss(21) + if len(f) < 22 { + return procStat{}, false + } + var st procStat + st.pgrp, _ = strconv.Atoi(f[2]) + ut, _ := strconv.ParseUint(f[11], 10, 64) + stt, _ := strconv.ParseUint(f[12], 10, 64) + st.jiffies = ut + stt + st.rssPages, _ = strconv.ParseInt(f[21], 10, 64) + return st, true +} + +// groups totals CPU time and memory per process group, in one pass over +// /proc. An engine is started as the leader of its own group, so its workers +// — vLLM starts several — are counted with it. +func (s *LoadSampler) groups() map[int]groupUse { + out := map[int]groupUse{} + s.eachProc(func(st procStat) { + g := out[st.pgrp] + g.jiffies += st.jiffies + g.rssPages += st.rssPages + out[st.pgrp] = g + }) + return out +} + +func (s *LoadSampler) eachProc(fn func(procStat)) { + entries, err := os.ReadDir(s.root) + if err != nil { + return + } + for _, e := range entries { + if _, err := strconv.Atoi(e.Name()); err != nil { + continue + } + if st, ok := readProcStat(filepath.Join(s.root, e.Name(), "stat")); ok { + fn(st) + } + } +} diff --git a/internal/agent/discovery/load_test.go b/internal/agent/discovery/load_test.go new file mode 100644 index 00000000..b9c1d711 --- /dev/null +++ b/internal/agent/discovery/load_test.go @@ -0,0 +1,96 @@ +package discovery + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" +) + +// fakeProc is a /proc with two CPUs, a memory picture and processes. +type fakeProc struct { + t *testing.T + root string +} + +func newFakeProc(t *testing.T) *fakeProc { + f := &fakeProc{t: t, root: t.TempDir()} + f.write("loadavg", "1.50 0.75 0.25 2/300 4242\n") + f.write("meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 3000000 kB\n") + return f +} + +func (f *fakeProc) write(name, body string) { + f.t.Helper() + p := filepath.Join(f.root, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + f.t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + f.t.Fatal(err) + } +} + +// cpu sets the aggregate counters: user and idle, the rest zero. +func (f *fakeProc) cpu(user, idle uint64) { + f.write("stat", fmt.Sprintf("cpu %d 0 0 %d 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n", user, idle)) +} + +// proc sets one process: its group, CPU jiffies (as utime) and RSS in pages. +func (f *fakeProc) proc(pid, pgrp int, jiffies uint64, rss int64) { + // The name holds a space and a parenthesis, as real ones can. + f.write(fmt.Sprintf("%d/stat", pid), fmt.Sprintf( + "%d (llama-server (x)) S 1 %d %d 0 -1 0 0 0 0 0 %d 0 0 0 20 0 1 0 100 1000 %d 0\n", + pid, pgrp, pgrp, jiffies, rss)) +} + +func TestLoadIsTheDifferenceBetweenTwoReadings(t *testing.T) { + p := newFakeProc(t) + at := time.Unix(1000, 0) + s := newLoadSampler(p.root, func() time.Time { return at }) + + p.cpu(100, 900) + p.proc(42, 42, 1000, 10) + p.proc(43, 42, 500, 5) // a worker in the engine's group + p.proc(7, 7, 99999, 1) // something else entirely + if _, ok := s.Sample(map[string]int{"shard-a": 42}); ok { + t.Fatal("the first reading has nothing to compare with and should report nothing") + } + + // Two seconds later: 300 of the 400 jiffies that passed were busy, and + // the engine's group used 400 of them. + at = at.Add(2 * time.Second) + p.cpu(400, 1000) + p.proc(42, 42, 1300, 10) + p.proc(43, 42, 600, 6) + l, ok := s.Sample(map[string]int{"shard-a": 42}) + if !ok { + t.Fatal("the second reading should report") + } + if l.CPUPercent != 75 || l.CPUCount != 2 { + t.Errorf("cpu %.1f%% of %d, want 75%% of 2", l.CPUPercent, l.CPUCount) + } + if l.Load1 != 1.5 || l.Load15 != 0.25 { + t.Errorf("load %v %v", l.Load1, l.Load15) + } + if l.MemUsed != 5000000*1024 || l.MemAvail != 3000000*1024 { + t.Errorf("memory used %d available %d", l.MemUsed, l.MemAvail) + } + if len(l.Shards) != 1 { + t.Fatalf("shards = %+v", l.Shards) + } + sh := l.Shards[0] + // 400 jiffies over 2 seconds at 100 a second is two cores. + if sh.ID != "shard-a" || sh.CPUCores != 2 || sh.RSSBytes != 16*int64(os.Getpagesize()) { + t.Errorf("shard = %+v", sh) + } +} + +func TestNoProcMeansNoLoad(t *testing.T) { + s := newLoadSampler(filepath.Join(t.TempDir(), "missing"), time.Now) + s.Sample(nil) + if _, ok := s.Sample(nil); ok { + t.Error("a system without /proc should report nothing, not zeros") + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 736095e0..2b69c9ab 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -207,6 +207,7 @@ func newServerCmd() *cobra.Command { web.NavExtras{History: history, Window: cfg.Monitoring.Window}, log)) adminMux.Handle("GET /_static/", web.Static()) adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version)) + adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log)) // The same page on the client-facing port at /admin, behind a // session and an admin flag. Handing the router the data rather diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 5cc73b4a..c81d71ff 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -115,6 +115,43 @@ type NodeView struct { LastSeenAgo string `json:"last_seen_ago"` AgentVersion string `json:"agent_version,omitempty"` Replicas int `json:"replicas"` + + // Load is how busy the machine is, absent when its agent has not + // measured it lately — which is unknown, not idle. + Load *LoadView `json:"load,omitempty"` +} + +// LoadView is a node's CPU and memory in use, at its last heartbeat. +type LoadView struct { + CPUPercent float64 `json:"cpu_percent"` + CPUs int `json:"cpus,omitempty"` + Load1 float64 `json:"load1"` + Load5 float64 `json:"load5"` + Load15 float64 `json:"load15"` + MemUsed int64 `json:"memory_used_bytes"` + MemTotal int64 `json:"memory_total_bytes"` + // MemPercent is used over used plus available. + MemPercent float64 `json:"memory_percent"` + AgeSec int64 `json:"age_seconds"` +} + +// loadFresh is how old a measurement may be and still be shown as current. +const loadFresh = time.Minute + +func loadView(l *inventory.Load, now time.Time) *LoadView { + if l == nil || now.Sub(l.At) > loadFresh { + return nil + } + v := &LoadView{ + CPUPercent: l.CPUPercent, CPUs: l.CPUCount, + Load1: l.Load1, Load5: l.Load5, Load15: l.Load15, + MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes, + AgeSec: int64(now.Sub(l.At).Seconds()), + } + if v.MemTotal > 0 { + v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal) + } + return v } // ModelView is a model as `ctl models` shows it. @@ -170,6 +207,10 @@ type ShardView struct { // waiting on one. It is what tells a replica pulling a hundred gigabytes // apart from one that is wedged — which look identical without it. Fetch *FetchView `json:"fetch,omitempty"` + // CPUCores and MemoryBytes are the engine's use of its node, from the + // node's last heartbeat; zero when not measured. + CPUCores float64 `json:"cpu_cores,omitempty"` + MemoryBytes int64 `json:"memory_bytes,omitempty"` } // FetchView is a download in flight. @@ -266,6 +307,7 @@ func (a *API) Nodes() []NodeView { if !n.LastSeen.IsZero() { v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String() } + v.Load = loadView(n.Load, time.Now()) out = append(out, v) } return out @@ -308,6 +350,12 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) { // Replicas builds the replica view, optionally filtered to one model. func (a *API) Replicas(model string) []ReplicaView { out := make([]ReplicaView, 0) + loads := map[string]*inventory.Load{} + for _, n := range a.ctl.Inventory().Nodes() { + if n.Load != nil && time.Since(n.Load.At) <= loadFresh { + loads[n.ID] = n.Load + } + } for _, rep := range a.ctl.Reconciler().Replicas() { if model != "" && rep.Model != model { continue @@ -338,6 +386,11 @@ func (a *API) Replicas(model string) []ReplicaView { sv.Fetch.Percent = &p } } + if l := loads[s.NodeID]; l != nil { + if use, ok := l.Shards[s.ID]; ok { + sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes + } + } v.Shards = append(v.Shards, sv) } out = append(out, v) diff --git a/internal/control/agentsvc/heartbeat_test.go b/internal/control/agentsvc/heartbeat_test.go index 67e39045..2ac47f6c 100644 --- a/internal/control/agentsvc/heartbeat_test.go +++ b/internal/control/agentsvc/heartbeat_test.go @@ -188,3 +188,25 @@ func TestALogRequestIsAnsweredByTheNodeItWasSentTo(t *testing.T) { t.Error("a node with no stream answered") } } + +func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) { + s := newTestService() + nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready}) + + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", Load: &llmgwv1.NodeLoad{ + CpuPercent: 62.5, CpuCount: 8, Load1: 3.2, MemoryUsedBytes: 6 << 30, MemoryAvailableBytes: 2 << 30, + Shards: []*llmgwv1.ShardLoad{{ShardId: "r1-0", CpuCores: 4.5, MemoryRssBytes: 5 << 30}}, + }}) + n, _ := s.inv.Get("n1") + if n.Load == nil || n.Load.CPUPercent != 62.5 || n.Load.MemUsedBytes != 6<<30 || + n.Load.Shards["r1-0"].CPUCores != 4.5 { + t.Fatalf("load = %+v", n.Load) + } + + // A heartbeat that measured nothing — the agent's first — leaves the last + // measurement standing rather than blanking it. + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"}) + if n, _ := s.inv.Get("n1"); n.Load == nil { + t.Error("a heartbeat without load erased it") + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 55ddaa88..e52b7bc1 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -309,7 +309,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { memMB := int(hb.GetMemoryTotalMb()) if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() && - !hb.GetProvisionChanged() { + !hb.GetProvisionChanged() && hb.GetLoad() == nil { return } @@ -400,9 +400,28 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { n.Provision = append(n.Provision, e) } } + if l := hb.GetLoad(); l != nil { + n.Load = loadFrom(l, time.Now()) + } s.inv.Upsert(n) } +// loadFrom is a heartbeat's load as the inventory keeps it. +func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { + out := &inventory.Load{ + At: at, CPUPercent: l.GetCpuPercent(), CPUCount: int(l.GetCpuCount()), + Load1: l.GetLoad1(), Load5: l.GetLoad5(), Load15: l.GetLoad15(), + MemUsedBytes: l.GetMemoryUsedBytes(), MemAvailBytes: l.GetMemoryAvailableBytes(), + Shards: map[string]inventory.ShardLoad{}, + } + for _, sh := range l.GetShards() { + out.Shards[sh.GetShardId()] = inventory.ShardLoad{ + CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(), + } + } + return out +} + func (s *Service) attach(ctx context.Context, nodeID string) *conn { c := &conn{ nodeID: nodeID, diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index b5b9c9ae..63273b23 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -59,8 +59,21 @@ type Collector struct { replicaState *prometheus.Desc replicaIdle *prometheus.Desc replicaReady *prometheus.Desc + + nodeCPU *prometheus.Desc + nodeCPUs *prometheus.Desc + nodeLoad *prometheus.Desc + nodeMemUsed *prometheus.Desc + nodeMemAvail *prometheus.Desc + shardCPU *prometheus.Desc + shardMemory *prometheus.Desc } +// loadFresh is how old a node's load may be and still be reported. A node +// that stopped sending it has an unknown load, and repeating its last figure +// would draw a busy machine long after it went away. +const loadFresh = time.Minute + // New builds the collector. func New(src Source) *Collector { node := []string{"node"} @@ -115,6 +128,25 @@ func New(src Source) *Collector { replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds", "Seconds since a replica was last sent a request. Absent when nothing has been reported, "+ "which is not the same as idle.", []string{"model", "replica"}, nil), + + // A node's own use, from its heartbeat. Absent when the agent has + // not measured it lately, which is not the same as idle. + nodeCPU: prometheus.NewDesc("llmgw_node_cpu_utilization_percent", + "How busy a node's CPUs were between its last two heartbeats, 0 to 100 across all cores.", node, nil), + nodeCPUs: prometheus.NewDesc("llmgw_node_cpus", + "CPUs a node has, so a load average can be read against it.", node, nil), + nodeLoad: prometheus.NewDesc("llmgw_node_load", + "The kernel's load average over 1, 5 and 15 minutes.", append(node, "window"), nil), + nodeMemUsed: prometheus.NewDesc("llmgw_node_memory_used_bytes", + "RAM in use: total less what the kernel could hand to a new process.", node, nil), + nodeMemAvail: prometheus.NewDesc("llmgw_node_memory_available_bytes", + "RAM the kernel could hand to a new process without swapping.", node, nil), + shardCPU: prometheus.NewDesc("llmgw_shard_cpu_cores", + "Cores an engine process group kept busy, the way top counts: 2.5 is two and a half.", + []string{"node", "model", "replica", "shard"}, nil), + shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes", + "Resident memory of an engine process group.", + []string{"node", "model", "replica", "shard"}, nil), } } @@ -123,6 +155,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { for _, d := range []*prometheus.Desc{ c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, c.nodeDiskFree, c.nodeDiskSize, + c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -135,6 +168,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { func (c *Collector) Collect(ch chan<- prometheus.Metric) { now := time.Now() + loads := map[string]*inventory.Load{} for _, n := range c.src.Nodes() { fabric := "" if len(n.Fabrics) > 0 { @@ -165,6 +199,21 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { float64(total), n.ID) } + if l := n.Load; l != nil && now.Sub(l.At) < loadFresh { + ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID) + if l.CPUCount > 0 { + ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID) + } + for w, v := range map[string]float64{"1m": l.Load1, "5m": l.Load5, "15m": l.Load15} { + ch <- prometheus.MustNewConstMetric(c.nodeLoad, prometheus.GaugeValue, v, n.ID, w) + } + if l.MemUsedBytes > 0 || l.MemAvailBytes > 0 { + ch <- prometheus.MustNewConstMetric(c.nodeMemUsed, prometheus.GaugeValue, float64(l.MemUsedBytes), n.ID) + ch <- prometheus.MustNewConstMetric(c.nodeMemAvail, prometheus.GaugeValue, float64(l.MemAvailBytes), n.ID) + } + loads[n.ID] = l + } + byClass := map[string]int{} for _, g := range n.GPUs { class := g.Class() @@ -207,6 +256,17 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { counts := map[[2]string]int{} ready := map[string]int{} for _, rep := range c.src.Replicas() { + // Each engine's share of its node, labelled with what it serves, so + // "which model is eating this machine" is one query. + for _, sh := range rep.Shards { + if l := loads[sh.NodeID]; l != nil { + if use, ok := l.Shards[sh.ID]; ok { + labels := []string{sh.NodeID, rep.Model, rep.ID, sh.ID} + ch <- prometheus.MustNewConstMetric(c.shardCPU, prometheus.GaugeValue, use.CPUCores, labels...) + ch <- prometheus.MustNewConstMetric(c.shardMemory, prometheus.GaugeValue, float64(use.RSSBytes), labels...) + } + } + } counts[[2]string{rep.Model, rep.State.String()}]++ if rep.State == reconciler.Ready { ready[rep.Model]++ diff --git a/internal/control/fleetmetrics/fleetmetrics_test.go b/internal/control/fleetmetrics/fleetmetrics_test.go index e0d158a9..cfab934b 100644 --- a/internal/control/fleetmetrics/fleetmetrics_test.go +++ b/internal/control/fleetmetrics/fleetmetrics_test.go @@ -139,3 +139,40 @@ func TestAnEmptyFleetCollectsWithoutError(t *testing.T) { t.Log("no metrics for an empty fleet, which is correct") } } + +func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) { + // The nodes are CPU machines: without this, how busy one was had no + // answer anywhere. + n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{ + At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1, + MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30, + Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}}, + }} + src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{ + ID: "r1", Model: "qwen", State: reconciler.Ready, + Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}}, + }}} + out := gather(t, src) + for _, want := range []string{ + "llmgw_node_cpu_utilization_percent node=n1 -> 62.5", + "llmgw_node_cpus node=n1 -> 8", + "llmgw_node_load node=n1 window=1m -> 3.5", + "llmgw_node_memory_used_bytes node=n1 -> 6442450944", + "llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5", + "llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } +} + +func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) { + // A node that stopped reporting has an unknown load. Repeating its last + // figure would draw a busy machine long after it went away. + n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), + Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}} + if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") { + t.Errorf("a stale load was reported:\n%s", out) + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 46172866..f9ae0fd7 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -110,6 +110,31 @@ type Node struct { Host discovery.Host `json:"host"` // Provision is where installing each wanted engine stands on this node. Provision []EngineInstall `json:"provision,omitempty"` + + // Load is how busy the machine was at the last heartbeat that measured + // it. Nil before the first one, and from an agent too old to say. Not + // persisted: it describes a moment, and a restored one would be stale. + Load *Load `json:"-"` +} + +// Load is a node's use of its CPU and memory, and each engine's share. +type Load struct { + At time.Time + CPUPercent float64 + CPUCount int + Load1 float64 + Load5 float64 + Load15 float64 + MemUsedBytes int64 + MemAvailBytes int64 + // Shards is each engine's use, by shard ID. + Shards map[string]ShardLoad +} + +// ShardLoad is one engine's use: cores busy, and resident memory. +type ShardLoad struct { + CPUCores float64 + RSSBytes int64 } // EngineInstall is where one engine's installation stands on a node. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index ff372b71..95e667ca 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -164,6 +164,8 @@ func (r *Router) Handler() http.Handler { }, r.log))) mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage( web.NodeLogsPage(r.clusterView, false, version.Version))) + mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage( + web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log))) pages.RegisterAdminUsers(mux, r.store) // Quotas are written on the same tree the accounts page builds, so // they are mounted together. diff --git a/internal/web/node_page.go b/internal/web/node_page.go new file mode 100644 index 00000000..e33ec448 --- /dev/null +++ b/internal/web/node_page.go @@ -0,0 +1,167 @@ +package web + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "sort" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// nodePageData is one node: what it is, how busy it is now, and how busy it +// has been. +type nodePageData struct { + Nav Nav + OnAdminListener bool + Node adminapi.NodeView + Engines []nodeEngine + Charts *nodeCharts + HasHistory bool + Back, Logs string + // Self is this page's address, for the in-place refresh. + Self string + Version, Now string + RefreshSec int +} + +// nodeEngine is one engine running on the node, and its share of it. +type nodeEngine struct { + Replica, Model, Shard, State string + CPUCores float64 + MemoryBytes int64 + // CPUShare is its cores as a percentage of the machine, for the bar. + CPUShare float64 +} + +// nodeCharts are the node's recent history, from the metrics store. +type nodeCharts struct { + CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries + Window string + Err string +} + +// nodePanels are the charts, each a query scoped to this node. %s is the +// node's ID, quoted. +var nodePanels = []struct { + expr, nameBy, unit string + into func(*nodeCharts, TimeSeries) +}{ + // Divided to a fraction because that is what the "%" unit draws. + {`llmgw_node_cpu_utilization_percent{node=%s} / 100`, "node", "%", + func(c *nodeCharts, t TimeSeries) { c.CPU = t }}, + {`llmgw_node_memory_used_bytes{node=%s}`, "node", "bytes", + func(c *nodeCharts, t TimeSeries) { c.Memory = t }}, + {`llmgw_node_load{node=%s}`, "window", "", + func(c *nodeCharts, t TimeSeries) { c.Load = t }}, + {`sum by (model) (llmgw_shard_cpu_cores{node=%s})`, "model", "", + func(c *nodeCharts, t TimeSeries) { c.EngineCPU = t }}, + {`sum by (model) (llmgw_shard_memory_bytes{node=%s})`, "model", "bytes", + func(c *nodeCharts, t TimeSeries) { c.EngineMemory = t }}, +} + +// NodePage shows one node. Its charts come from the metrics store when one +// is configured; the current figures come from the node's last heartbeat +// either way. +func NodePage(src Source, history History, window time.Duration, onAdminListener bool, + version string, log *slog.Logger, +) http.HandlerFunc { + if window <= 0 { + window = time.Hour + } + return func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + d := nodePageData{ + Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener, + Back: "/admin", Logs: "/admin/nodes/" + id + "/logs", Self: "/admin/nodes/" + id, + HasHistory: history != nil, Version: version, RefreshSec: 15, + Now: time.Now().UTC().Format(time.RFC3339), + } + if onAdminListener { + d.Back, d.Logs, d.Self = "/", "/nodes/"+id+"/logs", "/nodes/"+id + } + found := false + for _, n := range src.Nodes() { + if n.ID == id { + d.Node, found = n, true + break + } + } + // Only a node the control plane knows reaches a query below, which + // is what keeps the ID a name rather than something typed into PromQL. + if !found { + http.NotFound(w, r) + return + } + + var replicas []string + for _, rep := range src.Replicas("") { + for _, sh := range rep.Shards { + if sh.Node != id { + continue + } + e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State, + CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes} + if l := d.Node.Load; l != nil && l.CPUs > 0 { + e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs) + } + d.Engines = append(d.Engines, e) + replicas = append(replicas, rep.ID) + } + } + sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores }) + + if history != nil { + d.Charts = loadNodeCharts(r.Context(), history, id, replicas, window, log) + } + noStore(w) + render(w, "node", d) + } +} + +func loadNodeCharts(ctx context.Context, h History, node string, replicas []string, + window time.Duration, log *slog.Logger, +) *nodeCharts { + step := window / 60 + out := &nodeCharts{Window: window.String()} + quoted := fmt.Sprintf("%q", node) + for _, p := range nodePanels { + series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), p.nameBy, window, step) + if err != nil { + if log != nil { + log.Warn("could not read a node's history", "node", node, "err", err) + } + out.Err = err.Error() + return out + } + p.into(out, LineChart(series, p.unit, 4)) + } + // The engines' queues, for the replicas that are here now. + if len(replicas) > 0 { + re := make([]string, len(replicas)) + for i, r := range replicas { + re[i] = regexpQuote(r) + } + expr := fmt.Sprintf(`llmgw_endpoint_waiting{replica=~%q}`, strings.Join(re, "|")) + if series, err := h.RangeQuery(ctx, expr, "replica", window, step); err == nil { + out.Queue = LineChart(series, "", 4) + } + } + return out +} + +// regexpQuote escapes what a replica ID could carry into a PromQL regex. +// IDs are letters, digits and dashes; this is for the day one is not. +func regexpQuote(s string) string { + var b strings.Builder + for _, r := range s { + if strings.ContainsRune(`\.+*?()|[]{}^$`, r) { + b.WriteRune('\\') + } + b.WriteRune(r) + } + return b.String() +} diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go new file mode 100644 index 00000000..607870a2 --- /dev/null +++ b/internal/web/node_page_test.go @@ -0,0 +1,86 @@ +package web + +import ( + "net/http" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/promq" +) + +// nodeMux mounts the page where it lives, so the path carries the node. +func nodeMux(src Source, h History) http.Handler { + mux := http.NewServeMux() + if h == nil { + mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, nil, time.Hour, false, "v1", nil)) + } else { + mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, h, time.Hour, false, "v1", nil)) + } + return mux +} + +// busySource is fakeSource with a node that has been measured. +type busySource struct{ fakeSource } + +func (busySource) Nodes() []adminapi.NodeView { + n := fakeSource{}.Nodes()[0] + n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1, + MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75} + return []adminapi.NodeView{n} +} + +func (busySource) Replicas(string) []adminapi.ReplicaView { + r := fakeSource{}.Replicas("") + r[0].Shards[0].CPUCores, r[0].Shards[0].MemoryBytes = 4, 5<<30 + return r +} + +func TestANodesPageShowsHowBusyItIsAndWhatIsUsingIt(t *testing.T) { + h := &fakeHistory{byExpr: map[string][]promq.Series{ + `llmgw_node_cpu_utilization_percent{node="llmgw-node01"} / 100`: {ts("llmgw-node01", 0.2, 0.6)}, + }} + code, body := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/llmgw-node01", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + for _, want := range []string{ + "62%", "CPU · 8 cores", "RAM in use", "3.25", + // The engine and its share: four of eight cores is half the machine. + "served", "4.0 cores", "width: 50%", + `hx-get="/admin/nodes/llmgw-node01"`, + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q", want) + } + } + // Every chart is about this node, and nothing else. + for _, q := range h.asked { + if !strings.Contains(q, `"llmgw-node01"`) && !strings.Contains(q, `served-1`) { + t.Errorf("a query not scoped to the node: %s", q) + } + } +} + +func TestAnUnknownNodeIsNotFound(t *testing.T) { + h := &fakeHistory{} + code, _ := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/nope", nil) + if code != http.StatusNotFound { + t.Errorf("status %d", code) + } + // Nothing was asked of the metrics store for a name nobody knows. + if len(h.asked) != 0 { + t.Errorf("queried for an unknown node: %v", h.asked) + } +} + +func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) { + _, body := get(t, nodeMux(fakeSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil) + if !strings.Contains(body, "not measured lately") { + t.Error("an unmeasured node should say so") + } + if !strings.Contains(body, "No metrics store is configured") { + t.Error("without a metrics store the page should say where history comes from") + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index c6729a57..4cda5168 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -534,3 +534,11 @@ table.repo-files td:last-child .opt { display: block; margin-bottom: .2rem; } .own-use .tile { min-width: 9rem; } /* Between what the cluster is and how it is being used, on the cluster page. */ hr.part { border: 0; border-top: 1px solid var(--line); margin: 2.25rem 0 .5rem; } +/* How full a node is, in the cluster page's node table and on its own page. */ +td.load { white-space: nowrap; min-width: 8.5rem; } +.minibar { display: inline-block; width: 3.2rem; height: .45rem; border-radius: 999px; background: var(--bg); + border: 1px solid var(--line); overflow: hidden; vertical-align: middle; margin-right: .4rem; } +.minibar > span { display: block; height: 100%; background: var(--accent); } +.minibar.warm > span { background: var(--warn); } +.minibar.hot > span { background: var(--bad); } +.load-v { font-variant-numeric: tabular-nums; font-size: .85rem; } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index cd703874..b0e63a6d 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -51,13 +51,16 @@

Nodes

{{if .Nodes}} - +
NodeStateGPUsRAMDiskEnginesClassesFabricsModelsReplicasLast seen
{{range .Nodes}} - + {{/* A dash, not an empty bar, when the agent has not measured it lately: + unknown is not idle. */}} + + - {{/* Free out of total, because the first is meaningless without the second, and an em dash where nothing was reported: an agent that was never asked about its disk must not read as a full one. */}} @@ -68,9 +71,9 @@ -{{if .Unhealthy}}{{end}} -{{range .Provision}}{{if ne .State "ready"}}{{end}}{{end}} -{{range .Warnings}}{{end}} +{{if .Unhealthy}}{{end}} +{{range .Provision}}{{if ne .State "ready"}}{{end}}{{end}} +{{range .Warnings}}{{end}} {{end}}
NodeStateCPURAMGPUsDiskEnginesClassesFabricsModelsReplicasLast seen
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} + {{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} logs {{.State}}{{with .Load}}{{template "minibar" pctbar .CPUPercent}}{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} of {{.CPUs}}{{end}}{{else}}—{{end}}{{with .Load}}{{template "minibar" pctbar .MemPercent}}{{bytes .MemUsed}} of {{bytes .MemTotal}}{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}} {{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{if .Models}}{{len .Models}}{{else}}—{{end}} {{.Replicas}} {{or .LastSeenAgo "—"}}
{{join .Unhealthy}}
{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}
{{.}}
{{join .Unhealthy}}
{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}
{{.}}
{{else}}

No agents have registered.

{{end}} @@ -263,3 +266,6 @@ exposed:

{{.note}}

{{end}} + +{{/* A small horizontal bar for a percentage, coloured as it fills. */}} +{{define "minibar"}}{{end}} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html new file mode 100644 index 00000000..71c480a3 --- /dev/null +++ b/internal/web/templates/node.html @@ -0,0 +1,69 @@ +{{define "node"}} + + + + +llmgw — {{.Node.ID}} +
+
+

{{.Node.ID}}

+

{{.Node.State}} + {{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}} + · last seen {{or .Node.LastSeenAgo "never"}} ago · logs · cluster

+ {{if .OnAdminListener}}{{else}}{{template "nav" .Nav}}{{end}} +
+ +{{/* Refreshed in place, like the cluster page. */}} +
+{{with .Node}} +
+ {{with .Load}} +
{{printf "%.0f" .CPUPercent}}%
CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}
{{template "minibar" pctbar .CPUPercent}}
+
{{bytes .MemUsed}}
RAM in use · of {{bytes .MemTotal}}
{{template "minibar" pctbar .MemPercent}}
+
{{printf "%.2f" .Load1}}
load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m
+ {{else}} +
—
CPU and RAM not measured lately
+ {{end}} + {{if .DiskTotalGB}}
{{.DiskFreeGB}}G
disk free · of {{.DiskTotalGB}}G
{{end}} + {{if .GPUTotal}}
{{.GPUFree}}/{{.GPUTotal}}
GPUs free
{{end}} +
+{{if .Warnings}}{{range .Warnings}}

{{.}}

{{end}}{{end}} +{{if .Unhealthy}}

{{join .Unhealthy}}

{{end}} +{{end}} + +

What runs here

+{{if .Engines}} + +{{range .Engines}} + + + + + +{{end}}
ModelReplicaStateCPUMemory
{{.Model}}{{.Replica}}{{.State}}{{template "minibar" pctbar .CPUShare}}{{printf "%.1f" .CPUCores}} cores{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}
+{{else}}

No engines are placed on this node.

{{end}} +

Engines: {{if .Node.Engines}}{{join .Node.Engines}}{{else}}none installed{{end}}{{if .Node.Models}} · {{len .Node.Models}} model file(s) on disk{{end}}.

+
+ +{{if .HasHistory}}{{with .Charts}} +

The last {{.Window}}

+{{if .Err}}

No history: {{.Err}}

{{else}} +
+ {{template "tschart" dict "t" .CPU "title" "CPU" "note" "All cores together, from the node's heartbeat."}} + {{template "tschart" dict "t" .Memory "title" "RAM in use" "note" "Total less what the kernel could hand to a new process."}} +
+
+ {{template "tschart" dict "t" .EngineCPU "title" "Cores busy, by model" "note" "Each engine's process group, the way top counts it."}} + {{template "tschart" dict "t" .EngineMemory "title" "Engine memory, by model" "note" "Resident memory: the weights and the KV cache, mostly."}} +
+
+ {{template "tschart" dict "t" .Load "title" "Load average" "note" "Above the core count means work is waiting for a CPU."}} + {{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started, on the replicas here now."}} +
+{{end}} +{{end}}{{else}} +

No metrics store is configured, so there is no history here; monitoring.prometheus in the server configuration adds it.

+{{end}} + + +
{{end}} diff --git a/internal/web/web.go b/internal/web/web.go index 35cc5748..0eb3b66c 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -81,6 +81,11 @@ var funcs = template.FuncMap{ return p }, "ago": ago, + // pctbar is a percentage as the minibar template takes it: whole, and + // held to 0..100 so a measurement a little over does not draw past its box. + "pctbar": func(v float64) struct{ P int } { + return struct{ P int }{int(max(0, min(v, 100)) + 0.5)} + }, // Spend is stored in micro-units so accumulation is exact; it is shown in // whole units because that is the number an operator set a budget in. // compact renders a large count the way someone says it aloud: 1.2k rather