Measure each node's CPU and memory, and give every node a page

The nodes are CPU machines, and nothing measured how busy one was: the
heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the
total. The agent now reads /proc between heartbeats - CPU across all
cores, load averages, memory used and available, and each engine's
process group's cores and resident memory - and sends it with the
heartbeat. Nothing new runs on the nodes and Prometheus needs no new
target: the control plane exports it with the rest of the fleet, as
llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load,
llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes,
llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more
than a minute old is left out rather than repeated.

The cluster page's node table shows CPU and RAM as bars, and each node
links to a page of its own: current CPU, RAM, load and disk, the engines
on it with their cores and memory, and an hour of CPU, RAM, load,
per-model cores and memory and queue depth from Prometheus. It refreshes
in place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 23:21:23 +02:00
1 parent b06fb9f51f
commit daed835d88
19 files changed
+1289 -102

No files matched your search

+298 -95
View File
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13, 0}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -1202,6 +1202,179 @@ func (x *LocalWeights) GetShared() bool {
return false
}
// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
//
// Carried on the heartbeat rather than scraped from the node: the control
// plane already exports the fleet from one place, and a node exporter per
// machine would be a scrape target, a port and a firewall rule each.
type NodeLoad struct {
state protoimpl.MessageState `protogen:"open.v1"`
// cpu_percent is the whole machine, 0 to 100 however many cores it has.
CpuPercent float64 `protobuf:"fixed64,1,opt,name=cpu_percent,json=cpuPercent,proto3" json:"cpu_percent,omitempty"`
CpuCount int32 `protobuf:"varint,2,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
// The kernel's run-queue averages over 1, 5 and 15 minutes.
Load1 float64 `protobuf:"fixed64,3,opt,name=load1,proto3" json:"load1,omitempty"`
Load5 float64 `protobuf:"fixed64,4,opt,name=load5,proto3" json:"load5,omitempty"`
Load15 float64 `protobuf:"fixed64,5,opt,name=load15,proto3" json:"load15,omitempty"`
// memory_used_bytes is total less available: what the machine could not
// hand to a new process without evicting something.
MemoryUsedBytes int64 `protobuf:"varint,6,opt,name=memory_used_bytes,json=memoryUsedBytes,proto3" json:"memory_used_bytes,omitempty"`
MemoryAvailableBytes int64 `protobuf:"varint,7,opt,name=memory_available_bytes,json=memoryAvailableBytes,proto3" json:"memory_available_bytes,omitempty"`
Shards []*ShardLoad `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *NodeLoad) Reset() {
*x = NodeLoad{}
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *NodeLoad) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*NodeLoad) ProtoMessage() {}
func (x *NodeLoad) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use NodeLoad.ProtoReflect.Descriptor instead.
func (*NodeLoad) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
}
func (x *NodeLoad) GetCpuPercent() float64 {
if x != nil {
return x.CpuPercent
}
return 0
}
func (x *NodeLoad) GetCpuCount() int32 {
if x != nil {
return x.CpuCount
}
return 0
}
func (x *NodeLoad) GetLoad1() float64 {
if x != nil {
return x.Load1
}
return 0
}
func (x *NodeLoad) GetLoad5() float64 {
if x != nil {
return x.Load5
}
return 0
}
func (x *NodeLoad) GetLoad15() float64 {
if x != nil {
return x.Load15
}
return 0
}
func (x *NodeLoad) GetMemoryUsedBytes() int64 {
if x != nil {
return x.MemoryUsedBytes
}
return 0
}
func (x *NodeLoad) GetMemoryAvailableBytes() int64 {
if x != nil {
return x.MemoryAvailableBytes
}
return 0
}
func (x *NodeLoad) GetShards() []*ShardLoad {
if x != nil {
return x.Shards
}
return nil
}
// ShardLoad is one engine process group's use.
type ShardLoad struct {
state protoimpl.MessageState `protogen:"open.v1"`
ShardId string `protobuf:"bytes,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"`
// cpu_cores is in cores, the way top counts a process: 2.5 is two and a
// half cores busy, which a percentage of the machine would hide on a large
// one.
CpuCores float64 `protobuf:"fixed64,2,opt,name=cpu_cores,json=cpuCores,proto3" json:"cpu_cores,omitempty"`
MemoryRssBytes int64 `protobuf:"varint,3,opt,name=memory_rss_bytes,json=memoryRssBytes,proto3" json:"memory_rss_bytes,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ShardLoad) Reset() {
*x = ShardLoad{}
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ShardLoad) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ShardLoad) ProtoMessage() {}
func (x *ShardLoad) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead.
func (*ShardLoad) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
}
func (x *ShardLoad) GetShardId() string {
if x != nil {
return x.ShardId
}
return ""
}
func (x *ShardLoad) GetCpuCores() float64 {
if x != nil {
return x.CpuCores
}
return 0
}
func (x *ShardLoad) GetMemoryRssBytes() int64 {
if x != nil {
return x.MemoryRssBytes
}
return 0
}
type Heartbeat struct {
state protoimpl.MessageState `protogen:"open.v1"`
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
@@ -1238,13 +1411,17 @@ type Heartbeat struct {
// than "nothing to say", as for weights.
ProvisionChanged bool `protobuf:"varint,10,opt,name=provision_changed,json=provisionChanged,proto3" json:"provision_changed,omitempty"`
Provision []*ProvisionStatus `protobuf:"bytes,11,rep,name=provision,proto3" json:"provision,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// load is how busy the machine is, sampled over the interval since the
// previous heartbeat. Absent on the first one, which has nothing to take a
// difference from, and from an agent on a system without /proc.
Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1256,7 +1433,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1269,7 +1446,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
}
func (x *Heartbeat) GetNodeId() string {
@@ -1349,6 +1526,13 @@ func (x *Heartbeat) GetProvision() []*ProvisionStatus {
return nil
}
func (x *Heartbeat) GetLoad() *NodeLoad {
if x != nil {
return x.Load
}
return nil
}
// Provision is the software a node should have: which engines, at which
// versions. Nothing more.
//
@@ -1368,7 +1552,7 @@ type Provision struct {
func (x *Provision) Reset() {
*x = Provision{}
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1380,7 +1564,7 @@ func (x *Provision) String() string {
func (*Provision) ProtoMessage() {}
func (x *Provision) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1393,7 +1577,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message {
// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
func (*Provision) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
}
func (x *Provision) GetEngines() []*EngineWant {
@@ -1414,7 +1598,7 @@ type EngineWant struct {
func (x *EngineWant) Reset() {
*x = EngineWant{}
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1426,7 +1610,7 @@ func (x *EngineWant) String() string {
func (*EngineWant) ProtoMessage() {}
func (x *EngineWant) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1439,7 +1623,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
func (*EngineWant) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *EngineWant) GetEngine() string {
@@ -1472,7 +1656,7 @@ type ProvisionStatus struct {
func (x *ProvisionStatus) Reset() {
*x = ProvisionStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1484,7 +1668,7 @@ func (x *ProvisionStatus) String() string {
func (*ProvisionStatus) ProtoMessage() {}
func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1497,7 +1681,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
func (*ProvisionStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
}
func (x *ProvisionStatus) GetEngine() string {
@@ -1561,7 +1745,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1573,7 +1757,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1586,7 +1770,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
}
func (x *ShardStatus) GetNodeId() string {
@@ -1660,7 +1844,7 @@ type FetchProgress struct {
func (x *FetchProgress) Reset() {
*x = FetchProgress{}
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1672,7 +1856,7 @@ func (x *FetchProgress) String() string {
func (*FetchProgress) ProtoMessage() {}
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1685,7 +1869,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
func (*FetchProgress) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
}
func (x *FetchProgress) GetSource() string {
@@ -1741,7 +1925,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1753,7 +1937,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1766,7 +1950,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
}
func (x *StartShard) GetReplicaId() string {
@@ -1893,7 +2077,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1905,7 +2089,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1918,7 +2102,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
}
func (x *EngineProfile) GetName() string {
@@ -2014,7 +2198,7 @@ type RemoveWeights struct {
func (x *RemoveWeights) Reset() {
*x = RemoveWeights{}
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2026,7 +2210,7 @@ func (x *RemoveWeights) String() string {
func (*RemoveWeights) ProtoMessage() {}
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2039,7 +2223,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
func (*RemoveWeights) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
}
func (x *RemoveWeights) GetPaths() []string {
@@ -2059,7 +2243,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2071,7 +2255,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2084,7 +2268,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
}
func (x *StopShard) GetShardId() string {
@@ -2104,7 +2288,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2116,7 +2300,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2129,7 +2313,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
}
func (x *DrainShard) GetShardId() string {
@@ -2151,7 +2335,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2163,7 +2347,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2176,7 +2360,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
}
func (x *Ack) GetMessageId() string {
@@ -2220,7 +2404,7 @@ type LogsRequest struct {
func (x *LogsRequest) Reset() {
*x = LogsRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2232,7 +2416,7 @@ func (x *LogsRequest) String() string {
func (*LogsRequest) ProtoMessage() {}
func (x *LogsRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2245,7 +2429,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead.
func (*LogsRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
}
func (x *LogsRequest) GetRequestId() string {
@@ -2285,7 +2469,7 @@ type LogsReply struct {
func (x *LogsReply) Reset() {
*x = LogsReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2297,7 +2481,7 @@ func (x *LogsReply) String() string {
func (*LogsReply) ProtoMessage() {}
func (x *LogsReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2310,7 +2494,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use LogsReply.ProtoReflect.Descriptor instead.
func (*LogsReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
}
func (x *LogsReply) GetRequestId() string {
@@ -2448,7 +2632,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xc8\x04\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"\x9b\x02\n" +
"\bNodeLoad\x12\x1f\n" +
"\vcpu_percent\x18\x01 \x01(\x01R\n" +
"cpuPercent\x12\x1b\n" +
"\tcpu_count\x18\x02 \x01(\x05R\bcpuCount\x12\x14\n" +
"\x05load1\x18\x03 \x01(\x01R\x05load1\x12\x14\n" +
"\x05load5\x18\x04 \x01(\x01R\x05load5\x12\x16\n" +
"\x06load15\x18\x05 \x01(\x01R\x06load15\x12*\n" +
"\x11memory_used_bytes\x18\x06 \x01(\x03R\x0fmemoryUsedBytes\x124\n" +
"\x16memory_available_bytes\x18\a \x01(\x03R\x14memoryAvailableBytes\x12+\n" +
"\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\"m\n" +
"\tShardLoad\x12\x19\n" +
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\xf0\x04\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -2461,7 +2659,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x12+\n" +
"\x11provision_changed\x18\n" +
" \x01(\bR\x10provisionChanged\x127\n" +
"\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x1aA\n" +
"\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x12&\n" +
"\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
@@ -2584,7 +2783,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 27)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -2596,62 +2795,66 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*GPU)(nil), // 7: llmgw.v1.GPU
(*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment
(*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
(*Heartbeat)(nil), // 10: llmgw.v1.Heartbeat
(*Provision)(nil), // 11: llmgw.v1.Provision
(*EngineWant)(nil), // 12: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 13: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 14: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 15: llmgw.v1.FetchProgress
(*StartShard)(nil), // 16: llmgw.v1.StartShard
(*EngineProfile)(nil), // 17: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 18: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 19: llmgw.v1.StopShard
(*DrainShard)(nil), // 20: llmgw.v1.DrainShard
(*Ack)(nil), // 21: llmgw.v1.Ack
(*LogsRequest)(nil), // 22: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 23: llmgw.v1.LogsReply
nil, // 24: llmgw.v1.Register.LabelsEntry
nil, // 25: llmgw.v1.Register.EngineVersionsEntry
nil, // 26: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 27: llmgw.v1.EngineProfile.EnvEntry
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
(*Heartbeat)(nil), // 12: llmgw.v1.Heartbeat
(*Provision)(nil), // 13: llmgw.v1.Provision
(*EngineWant)(nil), // 14: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 15: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 16: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 17: llmgw.v1.FetchProgress
(*StartShard)(nil), // 18: llmgw.v1.StartShard
(*EngineProfile)(nil), // 19: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 20: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 21: llmgw.v1.StopShard
(*DrainShard)(nil), // 22: llmgw.v1.DrainShard
(*Ack)(nil), // 23: llmgw.v1.Ack
(*LogsRequest)(nil), // 24: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 25: llmgw.v1.LogsReply
nil, // 26: llmgw.v1.Register.LabelsEntry
nil, // 27: llmgw.v1.Register.EngineVersionsEntry
nil, // 28: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 29: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
10, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
14, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
23, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
16, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
19, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
20, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
21, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
18, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
11, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
22, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
24, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
12, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
16, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
25, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
18, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
21, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
22, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
23, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
20, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
13, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
24, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
26, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
25, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
27, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
5, // 20: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 21: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
26, // 22: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
13, // 23: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
12, // 24: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 25: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
15, // 26: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
17, // 27: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
27, // 28: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 29: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 30: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
30, // [30:31] is the sub-list for method output_type
29, // [29:30] is the sub-list for method input_type
29, // [29:29] is the sub-list for extension type_name
29, // [29:29] is the sub-list for extension extendee
0, // [0:29] is the sub-list for field type_name
11, // 20: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
5, // 21: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 22: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
28, // 23: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
15, // 24: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 25: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
14, // 26: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 27: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
17, // 28: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
19, // 29: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
29, // 30: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 31: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 32: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
32, // [32:33] is the sub-list for method output_type
31, // [31:32] is the sub-list for method input_type
31, // [31:31] is the sub-list for extension type_name
31, // [31:31] is the sub-list for extension extendee
0, // [0:31] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -2680,7 +2883,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 27,
NumMessages: 29,
NumExtensions: 0,
NumServices: 1,
},
+35
View File
@@ -231,6 +231,36 @@ message LocalWeights {
bool shared = 3;
}
// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
//
// Carried on the heartbeat rather than scraped from the node: the control
// plane already exports the fleet from one place, and a node exporter per
// machine would be a scrape target, a port and a firewall rule each.
message NodeLoad {
// cpu_percent is the whole machine, 0 to 100 however many cores it has.
double cpu_percent = 1;
int32 cpu_count = 2;
// The kernel's run-queue averages over 1, 5 and 15 minutes.
double load1 = 3;
double load5 = 4;
double load15 = 5;
// memory_used_bytes is total less available: what the machine could not
// hand to a new process without evicting something.
int64 memory_used_bytes = 6;
int64 memory_available_bytes = 7;
repeated ShardLoad shards = 8;
}
// ShardLoad is one engine process group's use.
message ShardLoad {
string shard_id = 1;
// cpu_cores is in cores, the way top counts a process: 2.5 is two and a
// half cores busy, which a percentage of the machine would hide on a large
// one.
double cpu_cores = 2;
int64 memory_rss_bytes = 3;
}
message Heartbeat {
string node_id = 1;
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
@@ -271,6 +301,11 @@ message Heartbeat {
// than "nothing to say", as for weights.
bool provision_changed = 10;
repeated ProvisionStatus provision = 11;
// load is how busy the machine is, sampled over the interval since the
// previous heartbeat. Absent on the first one, which has nothing to take a
// difference from, and from an agent on a system without /proc.
NodeLoad load = 12;
}
// Provision is the software a node should have: which engines, at which
+33
View File
@@ -352,6 +352,8 @@ type Agent struct {
// failover the old stream's heartbeat loop can still be running when the
// new one's starts.
lastWeights string
// load measures the machine between heartbeats.
load *discovery.LoadSampler
// fetchWatchers are the shards waiting on each source, so progress on one
// download reaches every shard that asked for it. A fetch is single-flight
@@ -396,6 +398,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
provStatus: map[string]provision.Status{},
logRing: ring,
provKick: make(chan struct{}, 1),
load: discovery.NewLoadSampler(),
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -883,9 +886,39 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
})
}
}
hb.Load = a.sampleLoad()
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
}
// sampleLoad measures the machine and each running engine since the last
// heartbeat. Nil on the first, which has nothing to compare with.
func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
if a.load == nil {
return nil
}
pids := map[string]int{}
for _, st := range a.sup.Statuses() {
if st.PID > 0 {
pids[st.ID] = st.PID
}
}
l, ok := a.load.Sample(pids)
if !ok {
return nil
}
out := &llmgwv1.NodeLoad{
CpuPercent: l.CPUPercent, CpuCount: int32(l.CPUCount),
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
MemoryUsedBytes: l.MemUsed, MemoryAvailableBytes: l.MemAvail,
}
for _, sh := range l.Shards {
out.Shards = append(out.Shards, &llmgwv1.ShardLoad{
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
})
}
return out
}
// weightsFingerprint is enough to notice a model appearing, disappearing or
// finishing its download.
//
+260
View File
@@ -0,0 +1,260 @@
package discovery
import (
"bufio"
"bytes"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// Load is how busy a node is: the machine as a whole, and each engine's share.
type Load struct {
CPUPercent float64
CPUCount int
Load1 float64
Load5 float64
Load15 float64
MemUsed int64
MemAvail int64
Shards []ShardLoad
}
// ShardLoad is one engine process group's use.
type ShardLoad struct {
ID string
CPUCores float64
RSSBytes int64
}
// userHZ is the unit /proc counts CPU time in. Fixed at 100 by the kernel's
// ABI on every architecture a node runs, whatever the internal tick rate.
const userHZ = 100
// LoadSampler measures load as the difference between two readings of /proc.
//
// CPU time is a counter, so a percentage needs two samples and the time
// between them. The sampler keeps the previous one; the first call has
// nothing to compare with and reports no load at all, rather than a zero that
// would read as an idle machine.
//
// Linux only, like every node. Elsewhere the files are missing and it reports
// nothing, as an older agent does.
type LoadSampler struct {
root string
now func() time.Time
mu sync.Mutex
prevAt time.Time
prevCPU cpuTimes
prevPG map[int]groupUse // process group -> what it has used
}
// groupUse is a process group's CPU time so far and its resident memory now.
type groupUse struct {
jiffies uint64
rssPages int64
}
type cpuTimes struct{ total, idle uint64 }
// NewLoadSampler reads the real /proc.
func NewLoadSampler() *LoadSampler { return newLoadSampler("/proc", time.Now) }
func newLoadSampler(root string, now func() time.Time) *LoadSampler {
return &LoadSampler{root: root, now: now}
}
// Sample reads the machine and the process groups led by the given shards'
// PIDs, keyed by shard ID. False when there is nothing to report yet.
func (s *LoadSampler) Sample(shards map[string]int) (Load, bool) {
cpu, ok := s.cpu()
if !ok {
return Load{}, false
}
groups := s.groups()
at := s.now()
s.mu.Lock()
prevAt, prevCPU, prevPG := s.prevAt, s.prevCPU, s.prevPG
s.prevAt, s.prevCPU, s.prevPG = at, cpu, groups
s.mu.Unlock()
if prevAt.IsZero() || cpu.total <= prevCPU.total {
return Load{}, false
}
l := Load{CPUCount: s.cpuCount()}
dTotal := float64(cpu.total - prevCPU.total)
dIdle := float64(cpu.idle - prevCPU.idle)
l.CPUPercent = clamp((dTotal-dIdle)/dTotal*100, 0, 100)
l.Load1, l.Load5, l.Load15 = s.loadavg()
total, avail := s.meminfo()
if total > 0 {
l.MemUsed, l.MemAvail = total-avail, avail
}
secs := at.Sub(prevAt).Seconds()
for id, pid := range shards {
if pid <= 0 {
continue
}
now := groups[pid]
sl := ShardLoad{ID: id, RSSBytes: now.rssPages * int64(os.Getpagesize())}
if before, ok := prevPG[pid]; ok && secs > 0 && now.jiffies >= before.jiffies {
sl.CPUCores = float64(now.jiffies-before.jiffies) / userHZ / secs
}
l.Shards = append(l.Shards, sl)
}
return l, true
}
func clamp(v, lo, hi float64) float64 { return max(lo, min(v, hi)) }
// cpu reads the aggregate line of /proc/stat. Idle includes iowait: a core
// waiting on the disk is not doing work.
func (s *LoadSampler) cpu() (cpuTimes, bool) {
b, err := os.ReadFile(filepath.Join(s.root, "stat"))
if err != nil {
return cpuTimes{}, false
}
line, _, _ := bytes.Cut(b, []byte("\n"))
f := strings.Fields(string(line))
if len(f) < 5 || f[0] != "cpu" {
return cpuTimes{}, false
}
var t cpuTimes
for i, v := range f[1:] {
// guest and guest_nice are already counted in user and nice.
if i >= 8 {
break
}
n, _ := strconv.ParseUint(v, 10, 64)
t.total += n
if i == 3 || i == 4 {
t.idle += n
}
}
return t, true
}
func (s *LoadSampler) cpuCount() int {
b, err := os.ReadFile(filepath.Join(s.root, "stat"))
if err != nil {
return 0
}
n := 0
for _, line := range strings.Split(string(b), "\n") {
if strings.HasPrefix(line, "cpu") && len(line) > 3 && line[3] >= '0' && line[3] <= '9' {
n++
}
}
return n
}
func (s *LoadSampler) loadavg() (l1, l5, l15 float64) {
b, err := os.ReadFile(filepath.Join(s.root, "loadavg"))
if err != nil {
return 0, 0, 0
}
f := strings.Fields(string(b))
if len(f) < 3 {
return 0, 0, 0
}
l1, _ = strconv.ParseFloat(f[0], 64)
l5, _ = strconv.ParseFloat(f[1], 64)
l15, _ = strconv.ParseFloat(f[2], 64)
return l1, l5, l15
}
// meminfo returns MemTotal and MemAvailable in bytes.
func (s *LoadSampler) meminfo() (total, avail int64) {
f, err := os.Open(filepath.Join(s.root, "meminfo"))
if err != nil {
return 0, 0
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
k, v, ok := strings.Cut(sc.Text(), ":")
if !ok {
continue
}
fields := strings.Fields(v)
if len(fields) == 0 {
continue
}
kb, _ := strconv.ParseInt(fields[0], 10, 64)
switch k {
case "MemTotal":
total = kb * 1024
case "MemAvailable":
avail = kb * 1024
}
}
return total, avail
}
// procStat is what one /proc/<pid>/stat says that load needs.
type procStat struct {
pgrp int
jiffies uint64
rssPages int64
}
// readProcStat parses /proc/<pid>/stat. The command name is in parentheses
// and may itself hold spaces and parentheses, so fields are counted from the
// last closing one.
func readProcStat(path string) (procStat, bool) {
b, err := os.ReadFile(path)
if err != nil {
return procStat{}, false
}
i := bytes.LastIndexByte(b, ')')
if i < 0 {
return procStat{}, false
}
f := strings.Fields(string(b[i+1:]))
// After the name: state(0) ppid(1) pgrp(2) ... utime(11) stime(12) ... rss(21)
if len(f) < 22 {
return procStat{}, false
}
var st procStat
st.pgrp, _ = strconv.Atoi(f[2])
ut, _ := strconv.ParseUint(f[11], 10, 64)
stt, _ := strconv.ParseUint(f[12], 10, 64)
st.jiffies = ut + stt
st.rssPages, _ = strconv.ParseInt(f[21], 10, 64)
return st, true
}
// groups totals CPU time and memory per process group, in one pass over
// /proc. An engine is started as the leader of its own group, so its workers
// — vLLM starts several — are counted with it.
func (s *LoadSampler) groups() map[int]groupUse {
out := map[int]groupUse{}
s.eachProc(func(st procStat) {
g := out[st.pgrp]
g.jiffies += st.jiffies
g.rssPages += st.rssPages
out[st.pgrp] = g
})
return out
}
func (s *LoadSampler) eachProc(fn func(procStat)) {
entries, err := os.ReadDir(s.root)
if err != nil {
return
}
for _, e := range entries {
if _, err := strconv.Atoi(e.Name()); err != nil {
continue
}
if st, ok := readProcStat(filepath.Join(s.root, e.Name(), "stat")); ok {
fn(st)
}
}
}
+96
View File
@@ -0,0 +1,96 @@
package discovery
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
)
// fakeProc is a /proc with two CPUs, a memory picture and processes.
type fakeProc struct {
t *testing.T
root string
}
func newFakeProc(t *testing.T) *fakeProc {
f := &fakeProc{t: t, root: t.TempDir()}
f.write("loadavg", "1.50 0.75 0.25 2/300 4242\n")
f.write("meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 3000000 kB\n")
return f
}
func (f *fakeProc) write(name, body string) {
f.t.Helper()
p := filepath.Join(f.root, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
f.t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
f.t.Fatal(err)
}
}
// cpu sets the aggregate counters: user and idle, the rest zero.
func (f *fakeProc) cpu(user, idle uint64) {
f.write("stat", fmt.Sprintf("cpu %d 0 0 %d 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n", user, idle))
}
// proc sets one process: its group, CPU jiffies (as utime) and RSS in pages.
func (f *fakeProc) proc(pid, pgrp int, jiffies uint64, rss int64) {
// The name holds a space and a parenthesis, as real ones can.
f.write(fmt.Sprintf("%d/stat", pid), fmt.Sprintf(
"%d (llama-server (x)) S 1 %d %d 0 -1 0 0 0 0 0 %d 0 0 0 20 0 1 0 100 1000 %d 0\n",
pid, pgrp, pgrp, jiffies, rss))
}
func TestLoadIsTheDifferenceBetweenTwoReadings(t *testing.T) {
p := newFakeProc(t)
at := time.Unix(1000, 0)
s := newLoadSampler(p.root, func() time.Time { return at })
p.cpu(100, 900)
p.proc(42, 42, 1000, 10)
p.proc(43, 42, 500, 5) // a worker in the engine's group
p.proc(7, 7, 99999, 1) // something else entirely
if _, ok := s.Sample(map[string]int{"shard-a": 42}); ok {
t.Fatal("the first reading has nothing to compare with and should report nothing")
}
// Two seconds later: 300 of the 400 jiffies that passed were busy, and
// the engine's group used 400 of them.
at = at.Add(2 * time.Second)
p.cpu(400, 1000)
p.proc(42, 42, 1300, 10)
p.proc(43, 42, 600, 6)
l, ok := s.Sample(map[string]int{"shard-a": 42})
if !ok {
t.Fatal("the second reading should report")
}
if l.CPUPercent != 75 || l.CPUCount != 2 {
t.Errorf("cpu %.1f%% of %d, want 75%% of 2", l.CPUPercent, l.CPUCount)
}
if l.Load1 != 1.5 || l.Load15 != 0.25 {
t.Errorf("load %v %v", l.Load1, l.Load15)
}
if l.MemUsed != 5000000*1024 || l.MemAvail != 3000000*1024 {
t.Errorf("memory used %d available %d", l.MemUsed, l.MemAvail)
}
if len(l.Shards) != 1 {
t.Fatalf("shards = %+v", l.Shards)
}
sh := l.Shards[0]
// 400 jiffies over 2 seconds at 100 a second is two cores.
if sh.ID != "shard-a" || sh.CPUCores != 2 || sh.RSSBytes != 16*int64(os.Getpagesize()) {
t.Errorf("shard = %+v", sh)
}
}
func TestNoProcMeansNoLoad(t *testing.T) {
s := newLoadSampler(filepath.Join(t.TempDir(), "missing"), time.Now)
s.Sample(nil)
if _, ok := s.Sample(nil); ok {
t.Error("a system without /proc should report nothing, not zeros")
}
}
+1
View File
@@ -207,6 +207,7 @@ func newServerCmd() *cobra.Command {
web.NavExtras{History: history, Window: cfg.Monitoring.Window}, log))
adminMux.Handle("GET /_static/", web.Static())
adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version))
adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log))
// The same page on the client-facing port at /admin, behind a
// session and an admin flag. Handing the router the data rather
+53
View File
@@ -115,6 +115,43 @@ type NodeView struct {
LastSeenAgo string `json:"last_seen_ago"`
AgentVersion string `json:"agent_version,omitempty"`
Replicas int `json:"replicas"`
// Load is how busy the machine is, absent when its agent has not
// measured it lately — which is unknown, not idle.
Load *LoadView `json:"load,omitempty"`
}
// LoadView is a node's CPU and memory in use, at its last heartbeat.
type LoadView struct {
CPUPercent float64 `json:"cpu_percent"`
CPUs int `json:"cpus,omitempty"`
Load1 float64 `json:"load1"`
Load5 float64 `json:"load5"`
Load15 float64 `json:"load15"`
MemUsed int64 `json:"memory_used_bytes"`
MemTotal int64 `json:"memory_total_bytes"`
// MemPercent is used over used plus available.
MemPercent float64 `json:"memory_percent"`
AgeSec int64 `json:"age_seconds"`
}
// loadFresh is how old a measurement may be and still be shown as current.
const loadFresh = time.Minute
func loadView(l *inventory.Load, now time.Time) *LoadView {
if l == nil || now.Sub(l.At) > loadFresh {
return nil
}
v := &LoadView{
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
AgeSec: int64(now.Sub(l.At).Seconds()),
}
if v.MemTotal > 0 {
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
}
return v
}
// ModelView is a model as `ctl models` shows it.
@@ -170,6 +207,10 @@ type ShardView struct {
// waiting on one. It is what tells a replica pulling a hundred gigabytes
// apart from one that is wedged — which look identical without it.
Fetch *FetchView `json:"fetch,omitempty"`
// CPUCores and MemoryBytes are the engine's use of its node, from the
// node's last heartbeat; zero when not measured.
CPUCores float64 `json:"cpu_cores,omitempty"`
MemoryBytes int64 `json:"memory_bytes,omitempty"`
}
// FetchView is a download in flight.
@@ -266,6 +307,7 @@ func (a *API) Nodes() []NodeView {
if !n.LastSeen.IsZero() {
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
}
v.Load = loadView(n.Load, time.Now())
out = append(out, v)
}
return out
@@ -308,6 +350,12 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) {
// Replicas builds the replica view, optionally filtered to one model.
func (a *API) Replicas(model string) []ReplicaView {
out := make([]ReplicaView, 0)
loads := map[string]*inventory.Load{}
for _, n := range a.ctl.Inventory().Nodes() {
if n.Load != nil && time.Since(n.Load.At) <= loadFresh {
loads[n.ID] = n.Load
}
}
for _, rep := range a.ctl.Reconciler().Replicas() {
if model != "" && rep.Model != model {
continue
@@ -338,6 +386,11 @@ func (a *API) Replicas(model string) []ReplicaView {
sv.Fetch.Percent = &p
}
}
if l := loads[s.NodeID]; l != nil {
if use, ok := l.Shards[s.ID]; ok {
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
}
}
v.Shards = append(v.Shards, sv)
}
out = append(out, v)
@@ -188,3 +188,25 @@ func TestALogRequestIsAnsweredByTheNodeItWasSentTo(t *testing.T) {
t.Error("a node with no stream answered")
}
}
func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) {
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", Load: &llmgwv1.NodeLoad{
CpuPercent: 62.5, CpuCount: 8, Load1: 3.2, MemoryUsedBytes: 6 << 30, MemoryAvailableBytes: 2 << 30,
Shards: []*llmgwv1.ShardLoad{{ShardId: "r1-0", CpuCores: 4.5, MemoryRssBytes: 5 << 30}},
}})
n, _ := s.inv.Get("n1")
if n.Load == nil || n.Load.CPUPercent != 62.5 || n.Load.MemUsedBytes != 6<<30 ||
n.Load.Shards["r1-0"].CPUCores != 4.5 {
t.Fatalf("load = %+v", n.Load)
}
// A heartbeat that measured nothing — the agent's first — leaves the last
// measurement standing rather than blanking it.
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
if n, _ := s.inv.Get("n1"); n.Load == nil {
t.Error("a heartbeat without load erased it")
}
}
+20 -1
View File
@@ -309,7 +309,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
memMB := int(hb.GetMemoryTotalMb())
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
!hb.GetProvisionChanged() {
!hb.GetProvisionChanged() && hb.GetLoad() == nil {
return
}
@@ -400,9 +400,28 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
n.Provision = append(n.Provision, e)
}
}
if l := hb.GetLoad(); l != nil {
n.Load = loadFrom(l, time.Now())
}
s.inv.Upsert(n)
}
// loadFrom is a heartbeat's load as the inventory keeps it.
func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
out := &inventory.Load{
At: at, CPUPercent: l.GetCpuPercent(), CPUCount: int(l.GetCpuCount()),
Load1: l.GetLoad1(), Load5: l.GetLoad5(), Load15: l.GetLoad15(),
MemUsedBytes: l.GetMemoryUsedBytes(), MemAvailBytes: l.GetMemoryAvailableBytes(),
Shards: map[string]inventory.ShardLoad{},
}
for _, sh := range l.GetShards() {
out.Shards[sh.GetShardId()] = inventory.ShardLoad{
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
}
}
return out
}
func (s *Service) attach(ctx context.Context, nodeID string) *conn {
c := &conn{
nodeID: nodeID,
@@ -59,8 +59,21 @@ type Collector struct {
replicaState *prometheus.Desc
replicaIdle *prometheus.Desc
replicaReady *prometheus.Desc
nodeCPU *prometheus.Desc
nodeCPUs *prometheus.Desc
nodeLoad *prometheus.Desc
nodeMemUsed *prometheus.Desc
nodeMemAvail *prometheus.Desc
shardCPU *prometheus.Desc
shardMemory *prometheus.Desc
}
// loadFresh is how old a node's load may be and still be reported. A node
// that stopped sending it has an unknown load, and repeating its last figure
// would draw a busy machine long after it went away.
const loadFresh = time.Minute
// New builds the collector.
func New(src Source) *Collector {
node := []string{"node"}
@@ -115,6 +128,25 @@ func New(src Source) *Collector {
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
"which is not the same as idle.", []string{"model", "replica"}, nil),
// A node's own use, from its heartbeat. Absent when the agent has
// not measured it lately, which is not the same as idle.
nodeCPU: prometheus.NewDesc("llmgw_node_cpu_utilization_percent",
"How busy a node's CPUs were between its last two heartbeats, 0 to 100 across all cores.", node, nil),
nodeCPUs: prometheus.NewDesc("llmgw_node_cpus",
"CPUs a node has, so a load average can be read against it.", node, nil),
nodeLoad: prometheus.NewDesc("llmgw_node_load",
"The kernel's load average over 1, 5 and 15 minutes.", append(node, "window"), nil),
nodeMemUsed: prometheus.NewDesc("llmgw_node_memory_used_bytes",
"RAM in use: total less what the kernel could hand to a new process.", node, nil),
nodeMemAvail: prometheus.NewDesc("llmgw_node_memory_available_bytes",
"RAM the kernel could hand to a new process without swapping.", node, nil),
shardCPU: prometheus.NewDesc("llmgw_shard_cpu_cores",
"Cores an engine process group kept busy, the way top counts: 2.5 is two and a half.",
[]string{"node", "model", "replica", "shard"}, nil),
shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes",
"Resident memory of an engine process group.",
[]string{"node", "model", "replica", "shard"}, nil),
}
}
@@ -123,6 +155,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -135,6 +168,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
now := time.Now()
loads := map[string]*inventory.Load{}
for _, n := range c.src.Nodes() {
fabric := ""
if len(n.Fabrics) > 0 {
@@ -165,6 +199,21 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
float64(total), n.ID)
}
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
if l.CPUCount > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
}
for w, v := range map[string]float64{"1m": l.Load1, "5m": l.Load5, "15m": l.Load15} {
ch <- prometheus.MustNewConstMetric(c.nodeLoad, prometheus.GaugeValue, v, n.ID, w)
}
if l.MemUsedBytes > 0 || l.MemAvailBytes > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeMemUsed, prometheus.GaugeValue, float64(l.MemUsedBytes), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeMemAvail, prometheus.GaugeValue, float64(l.MemAvailBytes), n.ID)
}
loads[n.ID] = l
}
byClass := map[string]int{}
for _, g := range n.GPUs {
class := g.Class()
@@ -207,6 +256,17 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
counts := map[[2]string]int{}
ready := map[string]int{}
for _, rep := range c.src.Replicas() {
// Each engine's share of its node, labelled with what it serves, so
// "which model is eating this machine" is one query.
for _, sh := range rep.Shards {
if l := loads[sh.NodeID]; l != nil {
if use, ok := l.Shards[sh.ID]; ok {
labels := []string{sh.NodeID, rep.Model, rep.ID, sh.ID}
ch <- prometheus.MustNewConstMetric(c.shardCPU, prometheus.GaugeValue, use.CPUCores, labels...)
ch <- prometheus.MustNewConstMetric(c.shardMemory, prometheus.GaugeValue, float64(use.RSSBytes), labels...)
}
}
}
counts[[2]string{rep.Model, rep.State.String()}]++
if rep.State == reconciler.Ready {
ready[rep.Model]++
@@ -139,3 +139,40 @@ func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
t.Log("no metrics for an empty fleet, which is correct")
}
}
func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
// The nodes are CPU machines: without this, how busy one was had no
// answer anywhere.
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
}}
src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
ID: "r1", Model: "qwen", State: reconciler.Ready,
Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
}}}
out := gather(t, src)
for _, want := range []string{
"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
"llmgw_node_cpus node=n1 -> 8",
"llmgw_node_load node=n1 window=1m -> 3.5",
"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out)
}
}
}
func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
// A node that stopped reporting has an unknown load. Repeating its last
// figure would draw a busy machine long after it went away.
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
t.Errorf("a stale load was reported:\n%s", out)
}
}
+25
View File
@@ -110,6 +110,31 @@ type Node struct {
Host discovery.Host `json:"host"`
// Provision is where installing each wanted engine stands on this node.
Provision []EngineInstall `json:"provision,omitempty"`
// Load is how busy the machine was at the last heartbeat that measured
// it. Nil before the first one, and from an agent too old to say. Not
// persisted: it describes a moment, and a restored one would be stale.
Load *Load `json:"-"`
}
// Load is a node's use of its CPU and memory, and each engine's share.
type Load struct {
At time.Time
CPUPercent float64
CPUCount int
Load1 float64
Load5 float64
Load15 float64
MemUsedBytes int64
MemAvailBytes int64
// Shards is each engine's use, by shard ID.
Shards map[string]ShardLoad
}
// ShardLoad is one engine's use: cores busy, and resident memory.
type ShardLoad struct {
CPUCores float64
RSSBytes int64
}
// EngineInstall is where one engine's installation stands on a node.
+2
View File
@@ -164,6 +164,8 @@ func (r *Router) Handler() http.Handler {
}, r.log)))
mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage(
web.NodeLogsPage(r.clusterView, false, version.Version)))
mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage(
web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log)))
pages.RegisterAdminUsers(mux, r.store)
// Quotas are written on the same tree the accounts page builds, so
// they are mounted together.
+167
View File
@@ -0,0 +1,167 @@
package web
import (
"context"
"fmt"
"log/slog"
"net/http"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// nodePageData is one node: what it is, how busy it is now, and how busy it
// has been.
type nodePageData struct {
Nav Nav
OnAdminListener bool
Node adminapi.NodeView
Engines []nodeEngine
Charts *nodeCharts
HasHistory bool
Back, Logs string
// Self is this page's address, for the in-place refresh.
Self string
Version, Now string
RefreshSec int
}
// nodeEngine is one engine running on the node, and its share of it.
type nodeEngine struct {
Replica, Model, Shard, State string
CPUCores float64
MemoryBytes int64
// CPUShare is its cores as a percentage of the machine, for the bar.
CPUShare float64
}
// nodeCharts are the node's recent history, from the metrics store.
type nodeCharts struct {
CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries
Window string
Err string
}
// nodePanels are the charts, each a query scoped to this node. %s is the
// node's ID, quoted.
var nodePanels = []struct {
expr, nameBy, unit string
into func(*nodeCharts, TimeSeries)
}{
// Divided to a fraction because that is what the "%" unit draws.
{`llmgw_node_cpu_utilization_percent{node=%s} / 100`, "node", "%",
func(c *nodeCharts, t TimeSeries) { c.CPU = t }},
{`llmgw_node_memory_used_bytes{node=%s}`, "node", "bytes",
func(c *nodeCharts, t TimeSeries) { c.Memory = t }},
{`llmgw_node_load{node=%s}`, "window", "",
func(c *nodeCharts, t TimeSeries) { c.Load = t }},
{`sum by (model) (llmgw_shard_cpu_cores{node=%s})`, "model", "",
func(c *nodeCharts, t TimeSeries) { c.EngineCPU = t }},
{`sum by (model) (llmgw_shard_memory_bytes{node=%s})`, "model", "bytes",
func(c *nodeCharts, t TimeSeries) { c.EngineMemory = t }},
}
// NodePage shows one node. Its charts come from the metrics store when one
// is configured; the current figures come from the node's last heartbeat
// either way.
func NodePage(src Source, history History, window time.Duration, onAdminListener bool,
version string, log *slog.Logger,
) http.HandlerFunc {
if window <= 0 {
window = time.Hour
}
return func(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
d := nodePageData{
Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener,
Back: "/admin", Logs: "/admin/nodes/" + id + "/logs", Self: "/admin/nodes/" + id,
HasHistory: history != nil, Version: version, RefreshSec: 15,
Now: time.Now().UTC().Format(time.RFC3339),
}
if onAdminListener {
d.Back, d.Logs, d.Self = "/", "/nodes/"+id+"/logs", "/nodes/"+id
}
found := false
for _, n := range src.Nodes() {
if n.ID == id {
d.Node, found = n, true
break
}
}
// Only a node the control plane knows reaches a query below, which
// is what keeps the ID a name rather than something typed into PromQL.
if !found {
http.NotFound(w, r)
return
}
var replicas []string
for _, rep := range src.Replicas("") {
for _, sh := range rep.Shards {
if sh.Node != id {
continue
}
e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State,
CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes}
if l := d.Node.Load; l != nil && l.CPUs > 0 {
e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs)
}
d.Engines = append(d.Engines, e)
replicas = append(replicas, rep.ID)
}
}
sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores })
if history != nil {
d.Charts = loadNodeCharts(r.Context(), history, id, replicas, window, log)
}
noStore(w)
render(w, "node", d)
}
}
func loadNodeCharts(ctx context.Context, h History, node string, replicas []string,
window time.Duration, log *slog.Logger,
) *nodeCharts {
step := window / 60
out := &nodeCharts{Window: window.String()}
quoted := fmt.Sprintf("%q", node)
for _, p := range nodePanels {
series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), p.nameBy, window, step)
if err != nil {
if log != nil {
log.Warn("could not read a node's history", "node", node, "err", err)
}
out.Err = err.Error()
return out
}
p.into(out, LineChart(series, p.unit, 4))
}
// The engines' queues, for the replicas that are here now.
if len(replicas) > 0 {
re := make([]string, len(replicas))
for i, r := range replicas {
re[i] = regexpQuote(r)
}
expr := fmt.Sprintf(`llmgw_endpoint_waiting{replica=~%q}`, strings.Join(re, "|"))
if series, err := h.RangeQuery(ctx, expr, "replica", window, step); err == nil {
out.Queue = LineChart(series, "", 4)
}
}
return out
}
// regexpQuote escapes what a replica ID could carry into a PromQL regex.
// IDs are letters, digits and dashes; this is for the day one is not.
func regexpQuote(s string) string {
var b strings.Builder
for _, r := range s {
if strings.ContainsRune(`\.+*?()|[]{}^$`, r) {
b.WriteRune('\\')
}
b.WriteRune(r)
}
return b.String()
}
+86
View File
@@ -0,0 +1,86 @@
package web
import (
"net/http"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)
// nodeMux mounts the page where it lives, so the path carries the node.
func nodeMux(src Source, h History) http.Handler {
mux := http.NewServeMux()
if h == nil {
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, nil, time.Hour, false, "v1", nil))
} else {
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, h, time.Hour, false, "v1", nil))
}
return mux
}
// busySource is fakeSource with a node that has been measured.
type busySource struct{ fakeSource }
func (busySource) Nodes() []adminapi.NodeView {
n := fakeSource{}.Nodes()[0]
n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
return []adminapi.NodeView{n}
}
func (busySource) Replicas(string) []adminapi.ReplicaView {
r := fakeSource{}.Replicas("")
r[0].Shards[0].CPUCores, r[0].Shards[0].MemoryBytes = 4, 5<<30
return r
}
func TestANodesPageShowsHowBusyItIsAndWhatIsUsingIt(t *testing.T) {
h := &fakeHistory{byExpr: map[string][]promq.Series{
`llmgw_node_cpu_utilization_percent{node="llmgw-node01"} / 100`: {ts("llmgw-node01", 0.2, 0.6)},
}}
code, body := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/llmgw-node01", nil)
if code != http.StatusOK {
t.Fatalf("status %d", code)
}
for _, want := range []string{
"62%", "CPU · 8 cores", "RAM in use", "3.25",
// The engine and its share: four of eight cores is half the machine.
"<td>served</td>", "4.0 cores", "width: 50%",
`hx-get="/admin/nodes/llmgw-node01"`,
} {
if !strings.Contains(body, want) {
t.Errorf("missing %q", want)
}
}
// Every chart is about this node, and nothing else.
for _, q := range h.asked {
if !strings.Contains(q, `"llmgw-node01"`) && !strings.Contains(q, `served-1`) {
t.Errorf("a query not scoped to the node: %s", q)
}
}
}
func TestAnUnknownNodeIsNotFound(t *testing.T) {
h := &fakeHistory{}
code, _ := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/nope", nil)
if code != http.StatusNotFound {
t.Errorf("status %d", code)
}
// Nothing was asked of the metrics store for a name nobody knows.
if len(h.asked) != 0 {
t.Errorf("queried for an unknown node: %v", h.asked)
}
}
func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
_, body := get(t, nodeMux(fakeSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
if !strings.Contains(body, "not measured lately") {
t.Error("an unmeasured node should say so")
}
if !strings.Contains(body, "No metrics store is configured") {
t.Error("without a metrics store the page should say where history comes from")
}
}
+8
View File
@@ -534,3 +534,11 @@ table.repo-files td:last-child .opt { display: block; margin-bottom: .2rem; }
.own-use .tile { min-width: 9rem; }
/* Between what the cluster is and how it is being used, on the cluster page. */
hr.part { border: 0; border-top: 1px solid var(--line); margin: 2.25rem 0 .5rem; }
/* How full a node is, in the cluster page's node table and on its own page. */
td.load { white-space: nowrap; min-width: 8.5rem; }
.minibar { display: inline-block; width: 3.2rem; height: .45rem; border-radius: 999px; background: var(--bg);
border: 1px solid var(--line); overflow: hidden; vertical-align: middle; margin-right: .4rem; }
.minibar > span { display: block; height: 100%; background: var(--accent); }
.minibar.warm > span { background: var(--warn); }
.minibar.hot > span { background: var(--bad); }
.load-v { font-variant-numeric: tabular-nums; font-size: .85rem; }
+12 -6
View File
@@ -51,13 +51,16 @@
<h2>Nodes</h2>
{{if .Nodes}}
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
<table class="nodes"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th><th class="num">GPUs</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
{{range .Nodes}}<tr>
<td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}"><code>{{.ID}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
<td class="state state-{{.State}}">{{.State}}</td>
{{/* A dash, not an empty bar, when the agent has not measured it lately:
unknown is not idle. */}}
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} <span class="note">of {{.CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">of {{bytes .MemTotal}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>
<td class="num">{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}</td>
{{/* Free out of total, because the first is meaningless without the
second, and an em dash where nothing was reported: an agent that
was never asked about its disk must not read as a full one. */}}
@@ -68,9 +71,9 @@
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
<td class="num">{{.Replicas}}</td>
<td>{{or .LastSeenAgo "—"}}</td>
</tr>{{if .Unhealthy}}<tr><td colspan="11" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
{{range .Provision}}{{if ne .State "ready"}}<tr><td colspan="11" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
{{range .Warnings}}<tr><td colspan="11" class="note bad">{{.}}</td></tr>{{end}}
</tr>{{if .Unhealthy}}<tr><td colspan="12" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
{{range .Provision}}{{if ne .State "ready"}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
{{range .Warnings}}<tr><td colspan="12" class="note bad">{{.}}</td></tr>{{end}}
{{end}}</table>
{{else}}<p class="empty">No agents have registered.</p>{{end}}
@@ -263,3 +266,6 @@ exposed:
<p class="note">{{.note}}</p>
</div>
{{end}}
{{/* A small horizontal bar for a percentage, coloured as it fills. */}}
{{define "minibar"}}<span class="minibar{{if ge .P 90}} hot{{else if ge .P 75}} warm{{end}}"><span style="width: {{.P}}%"></span></span>{{end}}
+69
View File
@@ -0,0 +1,69 @@
{{define "node"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
<script src="/_static/htmx-2.0.11.min.js"></script>
<title>llmgw — {{.Node.ID}}</title><link rel="stylesheet" href="{{stylesheet}}">
</head><body><div class="wrap">
<header>
<h1>{{.Node.ID}}</h1>
<p class="sub"><span class="state state-{{.Node.State}}">{{.Node.State}}</span>
{{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}}
· last seen {{or .Node.LastSeenAgo "never"}} ago · <a href="{{.Logs}}">logs</a> · <a href="{{.Back}}">cluster</a></p>
{{if .OnAdminListener}}<nav><a href="{{.Back}}">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
</header>
{{/* Refreshed in place, like the cluster page. */}}
<div id="live" hx-get="{{.Self}}" hx-trigger="every {{.RefreshSec}}s [document.visibilityState === 'visible']" hx-select="#live" hx-swap="outerHTML">
{{with .Node}}
<div class="tiles">
{{with .Load}}
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
{{else}}
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
{{end}}
{{if .DiskTotalGB}}<div class="tile"><div class="v">{{.DiskFreeGB}}G</div><div class="k">disk free · of {{.DiskTotalGB}}G</div></div>{{end}}
{{if .GPUTotal}}<div class="tile"><div class="v">{{.GPUFree}}/{{.GPUTotal}}</div><div class="k">GPUs free</div></div>{{end}}
</div>
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{end}}
<h2>What runs here</h2>
{{if .Engines}}
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
{{range .Engines}}<tr>
<td>{{.Model}}</td>
<td><code>{{.Replica}}</code></td>
<td class="state state-{{.State}}">{{.State}}</td>
<td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{printf "%.1f" .CPUCores}} cores</span></td>
<td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td>
</tr>{{end}}</table>
{{else}}<p class="empty">No engines are placed on this node.</p>{{end}}
<p class="note">Engines: {{if .Node.Engines}}{{join .Node.Engines}}{{else}}none installed{{end}}{{if .Node.Models}} · {{len .Node.Models}} model file(s) on disk{{end}}.</p>
</div>
{{if .HasHistory}}{{with .Charts}}
<h2>The last {{.Window}}</h2>
{{if .Err}}<p class="note error-row">No history: {{.Err}}</p>{{else}}
<div class="chart-row">
{{template "tschart" dict "t" .CPU "title" "CPU" "note" "All cores together, from the node's heartbeat."}}
{{template "tschart" dict "t" .Memory "title" "RAM in use" "note" "Total less what the kernel could hand to a new process."}}
</div>
<div class="chart-row">
{{template "tschart" dict "t" .EngineCPU "title" "Cores busy, by model" "note" "Each engine's process group, the way top counts it."}}
{{template "tschart" dict "t" .EngineMemory "title" "Engine memory, by model" "note" "Resident memory: the weights and the KV cache, mostly."}}
</div>
<div class="chart-row">
{{template "tschart" dict "t" .Load "title" "Load average" "note" "Above the core count means work is waiting for a CPU."}}
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started, on the replicas here now."}}
</div>
{{end}}
{{end}}{{else}}
<p class="note">No metrics store is configured, so there is no history here; <code>monitoring.prometheus</code> in the server configuration adds it.</p>
{{end}}
<footer><span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span><span>{{.Now}}</span></footer>
</div></body></html>{{end}}
+5
View File
@@ -81,6 +81,11 @@ var funcs = template.FuncMap{
return p
},
"ago": ago,
// pctbar is a percentage as the minibar template takes it: whole, and
// held to 0..100 so a measurement a little over does not draw past its box.
"pctbar": func(v float64) struct{ P int } {
return struct{ P int }{int(max(0, min(v, 100)) + 0.5)}
},
// Spend is stored in micro-units so accumulation is exact; it is shown in
// whole units because that is the number an operator set a budget in.
// compact renders a large count the way someone says it aloud: 1.2k rather