Measure each node's CPU and memory, and give every node a page
The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b06fb9f51f
commit
daed835d88
19 files changed
+1289
-102
No files matched your search
+298
-95
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
|
||||
|
||||
// Deprecated: Use ShardStatus_State.Descriptor instead.
|
||||
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13, 0}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15, 0}
|
||||
}
|
||||
|
||||
// AgentMessage is anything a node tells the control plane.
|
||||
@@ -1202,6 +1202,179 @@ func (x *LocalWeights) GetShared() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
|
||||
//
|
||||
// Carried on the heartbeat rather than scraped from the node: the control
|
||||
// plane already exports the fleet from one place, and a node exporter per
|
||||
// machine would be a scrape target, a port and a firewall rule each.
|
||||
type NodeLoad struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// cpu_percent is the whole machine, 0 to 100 however many cores it has.
|
||||
CpuPercent float64 `protobuf:"fixed64,1,opt,name=cpu_percent,json=cpuPercent,proto3" json:"cpu_percent,omitempty"`
|
||||
CpuCount int32 `protobuf:"varint,2,opt,name=cpu_count,json=cpuCount,proto3" json:"cpu_count,omitempty"`
|
||||
// The kernel's run-queue averages over 1, 5 and 15 minutes.
|
||||
Load1 float64 `protobuf:"fixed64,3,opt,name=load1,proto3" json:"load1,omitempty"`
|
||||
Load5 float64 `protobuf:"fixed64,4,opt,name=load5,proto3" json:"load5,omitempty"`
|
||||
Load15 float64 `protobuf:"fixed64,5,opt,name=load15,proto3" json:"load15,omitempty"`
|
||||
// memory_used_bytes is total less available: what the machine could not
|
||||
// hand to a new process without evicting something.
|
||||
MemoryUsedBytes int64 `protobuf:"varint,6,opt,name=memory_used_bytes,json=memoryUsedBytes,proto3" json:"memory_used_bytes,omitempty"`
|
||||
MemoryAvailableBytes int64 `protobuf:"varint,7,opt,name=memory_available_bytes,json=memoryAvailableBytes,proto3" json:"memory_available_bytes,omitempty"`
|
||||
Shards []*ShardLoad `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *NodeLoad) Reset() {
|
||||
*x = NodeLoad{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *NodeLoad) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*NodeLoad) ProtoMessage() {}
|
||||
|
||||
func (x *NodeLoad) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use NodeLoad.ProtoReflect.Descriptor instead.
|
||||
func (*NodeLoad) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetCpuPercent() float64 {
|
||||
if x != nil {
|
||||
return x.CpuPercent
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetCpuCount() int32 {
|
||||
if x != nil {
|
||||
return x.CpuCount
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetLoad1() float64 {
|
||||
if x != nil {
|
||||
return x.Load1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetLoad5() float64 {
|
||||
if x != nil {
|
||||
return x.Load5
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetLoad15() float64 {
|
||||
if x != nil {
|
||||
return x.Load15
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetMemoryUsedBytes() int64 {
|
||||
if x != nil {
|
||||
return x.MemoryUsedBytes
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetMemoryAvailableBytes() int64 {
|
||||
if x != nil {
|
||||
return x.MemoryAvailableBytes
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *NodeLoad) GetShards() []*ShardLoad {
|
||||
if x != nil {
|
||||
return x.Shards
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
type ShardLoad struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
ShardId string `protobuf:"bytes,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"`
|
||||
// cpu_cores is in cores, the way top counts a process: 2.5 is two and a
|
||||
// half cores busy, which a percentage of the machine would hide on a large
|
||||
// one.
|
||||
CpuCores float64 `protobuf:"fixed64,2,opt,name=cpu_cores,json=cpuCores,proto3" json:"cpu_cores,omitempty"`
|
||||
MemoryRssBytes int64 `protobuf:"varint,3,opt,name=memory_rss_bytes,json=memoryRssBytes,proto3" json:"memory_rss_bytes,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *ShardLoad) Reset() {
|
||||
*x = ShardLoad{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *ShardLoad) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ShardLoad) ProtoMessage() {}
|
||||
|
||||
func (x *ShardLoad) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead.
|
||||
func (*ShardLoad) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
|
||||
}
|
||||
|
||||
func (x *ShardLoad) GetShardId() string {
|
||||
if x != nil {
|
||||
return x.ShardId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *ShardLoad) GetCpuCores() float64 {
|
||||
if x != nil {
|
||||
return x.CpuCores
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *ShardLoad) GetMemoryRssBytes() int64 {
|
||||
if x != nil {
|
||||
return x.MemoryRssBytes
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
|
||||
@@ -1238,13 +1411,17 @@ type Heartbeat struct {
|
||||
// than "nothing to say", as for weights.
|
||||
ProvisionChanged bool `protobuf:"varint,10,opt,name=provision_changed,json=provisionChanged,proto3" json:"provision_changed,omitempty"`
|
||||
Provision []*ProvisionStatus `protobuf:"bytes,11,rep,name=provision,proto3" json:"provision,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// load is how busy the machine is, sampled over the interval since the
|
||||
// previous heartbeat. Absent on the first one, which has nothing to take a
|
||||
// difference from, and from an agent on a system without /proc.
|
||||
Load *NodeLoad `protobuf:"bytes,12,opt,name=load,proto3" json:"load,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Heartbeat) Reset() {
|
||||
*x = Heartbeat{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1256,7 +1433,7 @@ func (x *Heartbeat) String() string {
|
||||
func (*Heartbeat) ProtoMessage() {}
|
||||
|
||||
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1269,7 +1446,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
|
||||
func (*Heartbeat) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetNodeId() string {
|
||||
@@ -1349,6 +1526,13 @@ func (x *Heartbeat) GetProvision() []*ProvisionStatus {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetLoad() *NodeLoad {
|
||||
if x != nil {
|
||||
return x.Load
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
// versions. Nothing more.
|
||||
//
|
||||
@@ -1368,7 +1552,7 @@ type Provision struct {
|
||||
|
||||
func (x *Provision) Reset() {
|
||||
*x = Provision{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1380,7 +1564,7 @@ func (x *Provision) String() string {
|
||||
func (*Provision) ProtoMessage() {}
|
||||
|
||||
func (x *Provision) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1393,7 +1577,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
|
||||
func (*Provision) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
|
||||
}
|
||||
|
||||
func (x *Provision) GetEngines() []*EngineWant {
|
||||
@@ -1414,7 +1598,7 @@ type EngineWant struct {
|
||||
|
||||
func (x *EngineWant) Reset() {
|
||||
*x = EngineWant{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1426,7 +1610,7 @@ func (x *EngineWant) String() string {
|
||||
func (*EngineWant) ProtoMessage() {}
|
||||
|
||||
func (x *EngineWant) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1439,7 +1623,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
|
||||
func (*EngineWant) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
|
||||
}
|
||||
|
||||
func (x *EngineWant) GetEngine() string {
|
||||
@@ -1472,7 +1656,7 @@ type ProvisionStatus struct {
|
||||
|
||||
func (x *ProvisionStatus) Reset() {
|
||||
*x = ProvisionStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1484,7 +1668,7 @@ func (x *ProvisionStatus) String() string {
|
||||
func (*ProvisionStatus) ProtoMessage() {}
|
||||
|
||||
func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1497,7 +1681,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
|
||||
func (*ProvisionStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
|
||||
}
|
||||
|
||||
func (x *ProvisionStatus) GetEngine() string {
|
||||
@@ -1561,7 +1745,7 @@ type ShardStatus struct {
|
||||
|
||||
func (x *ShardStatus) Reset() {
|
||||
*x = ShardStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1573,7 +1757,7 @@ func (x *ShardStatus) String() string {
|
||||
func (*ShardStatus) ProtoMessage() {}
|
||||
|
||||
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1586,7 +1770,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
|
||||
func (*ShardStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
|
||||
}
|
||||
|
||||
func (x *ShardStatus) GetNodeId() string {
|
||||
@@ -1660,7 +1844,7 @@ type FetchProgress struct {
|
||||
|
||||
func (x *FetchProgress) Reset() {
|
||||
*x = FetchProgress{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1672,7 +1856,7 @@ func (x *FetchProgress) String() string {
|
||||
func (*FetchProgress) ProtoMessage() {}
|
||||
|
||||
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1685,7 +1869,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
|
||||
func (*FetchProgress) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
|
||||
}
|
||||
|
||||
func (x *FetchProgress) GetSource() string {
|
||||
@@ -1741,7 +1925,7 @@ type StartShard struct {
|
||||
|
||||
func (x *StartShard) Reset() {
|
||||
*x = StartShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1753,7 +1937,7 @@ func (x *StartShard) String() string {
|
||||
func (*StartShard) ProtoMessage() {}
|
||||
|
||||
func (x *StartShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1766,7 +1950,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
|
||||
func (*StartShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
|
||||
}
|
||||
|
||||
func (x *StartShard) GetReplicaId() string {
|
||||
@@ -1893,7 +2077,7 @@ type EngineProfile struct {
|
||||
|
||||
func (x *EngineProfile) Reset() {
|
||||
*x = EngineProfile{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1905,7 +2089,7 @@ func (x *EngineProfile) String() string {
|
||||
func (*EngineProfile) ProtoMessage() {}
|
||||
|
||||
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1918,7 +2102,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
|
||||
func (*EngineProfile) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
|
||||
}
|
||||
|
||||
func (x *EngineProfile) GetName() string {
|
||||
@@ -2014,7 +2198,7 @@ type RemoveWeights struct {
|
||||
|
||||
func (x *RemoveWeights) Reset() {
|
||||
*x = RemoveWeights{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2026,7 +2210,7 @@ func (x *RemoveWeights) String() string {
|
||||
func (*RemoveWeights) ProtoMessage() {}
|
||||
|
||||
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2039,7 +2223,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
|
||||
func (*RemoveWeights) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
|
||||
}
|
||||
|
||||
func (x *RemoveWeights) GetPaths() []string {
|
||||
@@ -2059,7 +2243,7 @@ type StopShard struct {
|
||||
|
||||
func (x *StopShard) Reset() {
|
||||
*x = StopShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2071,7 +2255,7 @@ func (x *StopShard) String() string {
|
||||
func (*StopShard) ProtoMessage() {}
|
||||
|
||||
func (x *StopShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2084,7 +2268,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
|
||||
func (*StopShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
|
||||
}
|
||||
|
||||
func (x *StopShard) GetShardId() string {
|
||||
@@ -2104,7 +2288,7 @@ type DrainShard struct {
|
||||
|
||||
func (x *DrainShard) Reset() {
|
||||
*x = DrainShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2116,7 +2300,7 @@ func (x *DrainShard) String() string {
|
||||
func (*DrainShard) ProtoMessage() {}
|
||||
|
||||
func (x *DrainShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2129,7 +2313,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
|
||||
func (*DrainShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
|
||||
}
|
||||
|
||||
func (x *DrainShard) GetShardId() string {
|
||||
@@ -2151,7 +2335,7 @@ type Ack struct {
|
||||
|
||||
func (x *Ack) Reset() {
|
||||
*x = Ack{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2163,7 +2347,7 @@ func (x *Ack) String() string {
|
||||
func (*Ack) ProtoMessage() {}
|
||||
|
||||
func (x *Ack) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2176,7 +2360,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
|
||||
func (*Ack) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
|
||||
}
|
||||
|
||||
func (x *Ack) GetMessageId() string {
|
||||
@@ -2220,7 +2404,7 @@ type LogsRequest struct {
|
||||
|
||||
func (x *LogsRequest) Reset() {
|
||||
*x = LogsRequest{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2232,7 +2416,7 @@ func (x *LogsRequest) String() string {
|
||||
func (*LogsRequest) ProtoMessage() {}
|
||||
|
||||
func (x *LogsRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2245,7 +2429,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead.
|
||||
func (*LogsRequest) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
|
||||
}
|
||||
|
||||
func (x *LogsRequest) GetRequestId() string {
|
||||
@@ -2285,7 +2469,7 @@ type LogsReply struct {
|
||||
|
||||
func (x *LogsReply) Reset() {
|
||||
*x = LogsReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2297,7 +2481,7 @@ func (x *LogsReply) String() string {
|
||||
func (*LogsReply) ProtoMessage() {}
|
||||
|
||||
func (x *LogsReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2310,7 +2494,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LogsReply.ProtoReflect.Descriptor instead.
|
||||
func (*LogsReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
|
||||
}
|
||||
|
||||
func (x *LogsReply) GetRequestId() string {
|
||||
@@ -2448,7 +2632,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
|
||||
"\n" +
|
||||
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
|
||||
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xc8\x04\n" +
|
||||
"\x06shared\x18\x03 \x01(\bR\x06shared\"\x9b\x02\n" +
|
||||
"\bNodeLoad\x12\x1f\n" +
|
||||
"\vcpu_percent\x18\x01 \x01(\x01R\n" +
|
||||
"cpuPercent\x12\x1b\n" +
|
||||
"\tcpu_count\x18\x02 \x01(\x05R\bcpuCount\x12\x14\n" +
|
||||
"\x05load1\x18\x03 \x01(\x01R\x05load1\x12\x14\n" +
|
||||
"\x05load5\x18\x04 \x01(\x01R\x05load5\x12\x16\n" +
|
||||
"\x06load15\x18\x05 \x01(\x01R\x06load15\x12*\n" +
|
||||
"\x11memory_used_bytes\x18\x06 \x01(\x03R\x0fmemoryUsedBytes\x124\n" +
|
||||
"\x16memory_available_bytes\x18\a \x01(\x03R\x14memoryAvailableBytes\x12+\n" +
|
||||
"\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\"m\n" +
|
||||
"\tShardLoad\x12\x19\n" +
|
||||
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
|
||||
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\xf0\x04\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
@@ -2461,7 +2659,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x12+\n" +
|
||||
"\x11provision_changed\x18\n" +
|
||||
" \x01(\bR\x10provisionChanged\x127\n" +
|
||||
"\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x1aA\n" +
|
||||
"\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x12&\n" +
|
||||
"\x04load\x18\f \x01(\v2\x12.llmgw.v1.NodeLoadR\x04load\x1aA\n" +
|
||||
"\x13EngineVersionsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
|
||||
@@ -2584,7 +2783,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 27)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -2596,62 +2795,66 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*GPU)(nil), // 7: llmgw.v1.GPU
|
||||
(*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment
|
||||
(*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
|
||||
(*Heartbeat)(nil), // 10: llmgw.v1.Heartbeat
|
||||
(*Provision)(nil), // 11: llmgw.v1.Provision
|
||||
(*EngineWant)(nil), // 12: llmgw.v1.EngineWant
|
||||
(*ProvisionStatus)(nil), // 13: llmgw.v1.ProvisionStatus
|
||||
(*ShardStatus)(nil), // 14: llmgw.v1.ShardStatus
|
||||
(*FetchProgress)(nil), // 15: llmgw.v1.FetchProgress
|
||||
(*StartShard)(nil), // 16: llmgw.v1.StartShard
|
||||
(*EngineProfile)(nil), // 17: llmgw.v1.EngineProfile
|
||||
(*RemoveWeights)(nil), // 18: llmgw.v1.RemoveWeights
|
||||
(*StopShard)(nil), // 19: llmgw.v1.StopShard
|
||||
(*DrainShard)(nil), // 20: llmgw.v1.DrainShard
|
||||
(*Ack)(nil), // 21: llmgw.v1.Ack
|
||||
(*LogsRequest)(nil), // 22: llmgw.v1.LogsRequest
|
||||
(*LogsReply)(nil), // 23: llmgw.v1.LogsReply
|
||||
nil, // 24: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 25: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 26: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 27: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
|
||||
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
|
||||
(*Heartbeat)(nil), // 12: llmgw.v1.Heartbeat
|
||||
(*Provision)(nil), // 13: llmgw.v1.Provision
|
||||
(*EngineWant)(nil), // 14: llmgw.v1.EngineWant
|
||||
(*ProvisionStatus)(nil), // 15: llmgw.v1.ProvisionStatus
|
||||
(*ShardStatus)(nil), // 16: llmgw.v1.ShardStatus
|
||||
(*FetchProgress)(nil), // 17: llmgw.v1.FetchProgress
|
||||
(*StartShard)(nil), // 18: llmgw.v1.StartShard
|
||||
(*EngineProfile)(nil), // 19: llmgw.v1.EngineProfile
|
||||
(*RemoveWeights)(nil), // 20: llmgw.v1.RemoveWeights
|
||||
(*StopShard)(nil), // 21: llmgw.v1.StopShard
|
||||
(*DrainShard)(nil), // 22: llmgw.v1.DrainShard
|
||||
(*Ack)(nil), // 23: llmgw.v1.Ack
|
||||
(*LogsRequest)(nil), // 24: llmgw.v1.LogsRequest
|
||||
(*LogsReply)(nil), // 25: llmgw.v1.LogsReply
|
||||
nil, // 26: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 27: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 28: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 29: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
10, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
|
||||
14, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
23, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
16, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
19, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
20, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
21, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
18, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
11, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
22, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
24, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
12, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
|
||||
16, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
25, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
18, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
21, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
22, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
23, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
20, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
13, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
24, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
26, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
25, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
27, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
5, // 20: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 21: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
26, // 22: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
13, // 23: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
12, // 24: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 25: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
15, // 26: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
17, // 27: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
27, // 28: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 29: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 30: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
30, // [30:31] is the sub-list for method output_type
|
||||
29, // [29:30] is the sub-list for method input_type
|
||||
29, // [29:29] is the sub-list for extension type_name
|
||||
29, // [29:29] is the sub-list for extension extendee
|
||||
0, // [0:29] is the sub-list for field type_name
|
||||
11, // 20: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
5, // 21: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 22: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
28, // 23: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
15, // 24: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 25: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
14, // 26: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 27: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
17, // 28: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
19, // 29: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
29, // 30: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 31: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 32: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
32, // [32:33] is the sub-list for method output_type
|
||||
31, // [31:32] is the sub-list for method input_type
|
||||
31, // [31:31] is the sub-list for extension type_name
|
||||
31, // [31:31] is the sub-list for extension extendee
|
||||
0, // [0:31] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -2680,7 +2883,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 27,
|
||||
NumMessages: 29,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -231,6 +231,36 @@ message LocalWeights {
|
||||
bool shared = 3;
|
||||
}
|
||||
|
||||
// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
|
||||
//
|
||||
// Carried on the heartbeat rather than scraped from the node: the control
|
||||
// plane already exports the fleet from one place, and a node exporter per
|
||||
// machine would be a scrape target, a port and a firewall rule each.
|
||||
message NodeLoad {
|
||||
// cpu_percent is the whole machine, 0 to 100 however many cores it has.
|
||||
double cpu_percent = 1;
|
||||
int32 cpu_count = 2;
|
||||
// The kernel's run-queue averages over 1, 5 and 15 minutes.
|
||||
double load1 = 3;
|
||||
double load5 = 4;
|
||||
double load15 = 5;
|
||||
// memory_used_bytes is total less available: what the machine could not
|
||||
// hand to a new process without evicting something.
|
||||
int64 memory_used_bytes = 6;
|
||||
int64 memory_available_bytes = 7;
|
||||
repeated ShardLoad shards = 8;
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
message ShardLoad {
|
||||
string shard_id = 1;
|
||||
// cpu_cores is in cores, the way top counts a process: 2.5 is two and a
|
||||
// half cores busy, which a percentage of the machine would hide on a large
|
||||
// one.
|
||||
double cpu_cores = 2;
|
||||
int64 memory_rss_bytes = 3;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
string node_id = 1;
|
||||
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
||||
@@ -271,6 +301,11 @@ message Heartbeat {
|
||||
// than "nothing to say", as for weights.
|
||||
bool provision_changed = 10;
|
||||
repeated ProvisionStatus provision = 11;
|
||||
|
||||
// load is how busy the machine is, sampled over the interval since the
|
||||
// previous heartbeat. Absent on the first one, which has nothing to take a
|
||||
// difference from, and from an agent on a system without /proc.
|
||||
NodeLoad load = 12;
|
||||
}
|
||||
|
||||
// Provision is the software a node should have: which engines, at which
|
||||
|
||||
@@ -352,6 +352,8 @@ type Agent struct {
|
||||
// failover the old stream's heartbeat loop can still be running when the
|
||||
// new one's starts.
|
||||
lastWeights string
|
||||
// load measures the machine between heartbeats.
|
||||
load *discovery.LoadSampler
|
||||
|
||||
// fetchWatchers are the shards waiting on each source, so progress on one
|
||||
// download reaches every shard that asked for it. A fetch is single-flight
|
||||
@@ -396,6 +398,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
provStatus: map[string]provision.Status{},
|
||||
logRing: ring,
|
||||
provKick: make(chan struct{}, 1),
|
||||
load: discovery.NewLoadSampler(),
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
return a, nil
|
||||
@@ -883,9 +886,39 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
})
|
||||
}
|
||||
}
|
||||
hb.Load = a.sampleLoad()
|
||||
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
|
||||
}
|
||||
|
||||
// sampleLoad measures the machine and each running engine since the last
|
||||
// heartbeat. Nil on the first, which has nothing to compare with.
|
||||
func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
|
||||
if a.load == nil {
|
||||
return nil
|
||||
}
|
||||
pids := map[string]int{}
|
||||
for _, st := range a.sup.Statuses() {
|
||||
if st.PID > 0 {
|
||||
pids[st.ID] = st.PID
|
||||
}
|
||||
}
|
||||
l, ok := a.load.Sample(pids)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := &llmgwv1.NodeLoad{
|
||||
CpuPercent: l.CPUPercent, CpuCount: int32(l.CPUCount),
|
||||
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
||||
MemoryUsedBytes: l.MemUsed, MemoryAvailableBytes: l.MemAvail,
|
||||
}
|
||||
for _, sh := range l.Shards {
|
||||
out.Shards = append(out.Shards, &llmgwv1.ShardLoad{
|
||||
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// weightsFingerprint is enough to notice a model appearing, disappearing or
|
||||
// finishing its download.
|
||||
//
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Load is how busy a node is: the machine as a whole, and each engine's share.
|
||||
type Load struct {
|
||||
CPUPercent float64
|
||||
CPUCount int
|
||||
Load1 float64
|
||||
Load5 float64
|
||||
Load15 float64
|
||||
MemUsed int64
|
||||
MemAvail int64
|
||||
Shards []ShardLoad
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
type ShardLoad struct {
|
||||
ID string
|
||||
CPUCores float64
|
||||
RSSBytes int64
|
||||
}
|
||||
|
||||
// userHZ is the unit /proc counts CPU time in. Fixed at 100 by the kernel's
|
||||
// ABI on every architecture a node runs, whatever the internal tick rate.
|
||||
const userHZ = 100
|
||||
|
||||
// LoadSampler measures load as the difference between two readings of /proc.
|
||||
//
|
||||
// CPU time is a counter, so a percentage needs two samples and the time
|
||||
// between them. The sampler keeps the previous one; the first call has
|
||||
// nothing to compare with and reports no load at all, rather than a zero that
|
||||
// would read as an idle machine.
|
||||
//
|
||||
// Linux only, like every node. Elsewhere the files are missing and it reports
|
||||
// nothing, as an older agent does.
|
||||
type LoadSampler struct {
|
||||
root string
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
prevAt time.Time
|
||||
prevCPU cpuTimes
|
||||
prevPG map[int]groupUse // process group -> what it has used
|
||||
}
|
||||
|
||||
// groupUse is a process group's CPU time so far and its resident memory now.
|
||||
type groupUse struct {
|
||||
jiffies uint64
|
||||
rssPages int64
|
||||
}
|
||||
|
||||
type cpuTimes struct{ total, idle uint64 }
|
||||
|
||||
// NewLoadSampler reads the real /proc.
|
||||
func NewLoadSampler() *LoadSampler { return newLoadSampler("/proc", time.Now) }
|
||||
|
||||
func newLoadSampler(root string, now func() time.Time) *LoadSampler {
|
||||
return &LoadSampler{root: root, now: now}
|
||||
}
|
||||
|
||||
// Sample reads the machine and the process groups led by the given shards'
|
||||
// PIDs, keyed by shard ID. False when there is nothing to report yet.
|
||||
func (s *LoadSampler) Sample(shards map[string]int) (Load, bool) {
|
||||
cpu, ok := s.cpu()
|
||||
if !ok {
|
||||
return Load{}, false
|
||||
}
|
||||
groups := s.groups()
|
||||
at := s.now()
|
||||
|
||||
s.mu.Lock()
|
||||
prevAt, prevCPU, prevPG := s.prevAt, s.prevCPU, s.prevPG
|
||||
s.prevAt, s.prevCPU, s.prevPG = at, cpu, groups
|
||||
s.mu.Unlock()
|
||||
|
||||
if prevAt.IsZero() || cpu.total <= prevCPU.total {
|
||||
return Load{}, false
|
||||
}
|
||||
l := Load{CPUCount: s.cpuCount()}
|
||||
dTotal := float64(cpu.total - prevCPU.total)
|
||||
dIdle := float64(cpu.idle - prevCPU.idle)
|
||||
l.CPUPercent = clamp((dTotal-dIdle)/dTotal*100, 0, 100)
|
||||
l.Load1, l.Load5, l.Load15 = s.loadavg()
|
||||
total, avail := s.meminfo()
|
||||
if total > 0 {
|
||||
l.MemUsed, l.MemAvail = total-avail, avail
|
||||
}
|
||||
|
||||
secs := at.Sub(prevAt).Seconds()
|
||||
for id, pid := range shards {
|
||||
if pid <= 0 {
|
||||
continue
|
||||
}
|
||||
now := groups[pid]
|
||||
sl := ShardLoad{ID: id, RSSBytes: now.rssPages * int64(os.Getpagesize())}
|
||||
if before, ok := prevPG[pid]; ok && secs > 0 && now.jiffies >= before.jiffies {
|
||||
sl.CPUCores = float64(now.jiffies-before.jiffies) / userHZ / secs
|
||||
}
|
||||
l.Shards = append(l.Shards, sl)
|
||||
}
|
||||
return l, true
|
||||
}
|
||||
|
||||
func clamp(v, lo, hi float64) float64 { return max(lo, min(v, hi)) }
|
||||
|
||||
// cpu reads the aggregate line of /proc/stat. Idle includes iowait: a core
|
||||
// waiting on the disk is not doing work.
|
||||
func (s *LoadSampler) cpu() (cpuTimes, bool) {
|
||||
b, err := os.ReadFile(filepath.Join(s.root, "stat"))
|
||||
if err != nil {
|
||||
return cpuTimes{}, false
|
||||
}
|
||||
line, _, _ := bytes.Cut(b, []byte("\n"))
|
||||
f := strings.Fields(string(line))
|
||||
if len(f) < 5 || f[0] != "cpu" {
|
||||
return cpuTimes{}, false
|
||||
}
|
||||
var t cpuTimes
|
||||
for i, v := range f[1:] {
|
||||
// guest and guest_nice are already counted in user and nice.
|
||||
if i >= 8 {
|
||||
break
|
||||
}
|
||||
n, _ := strconv.ParseUint(v, 10, 64)
|
||||
t.total += n
|
||||
if i == 3 || i == 4 {
|
||||
t.idle += n
|
||||
}
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
func (s *LoadSampler) cpuCount() int {
|
||||
b, err := os.ReadFile(filepath.Join(s.root, "stat"))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
if strings.HasPrefix(line, "cpu") && len(line) > 3 && line[3] >= '0' && line[3] <= '9' {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (s *LoadSampler) loadavg() (l1, l5, l15 float64) {
|
||||
b, err := os.ReadFile(filepath.Join(s.root, "loadavg"))
|
||||
if err != nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
f := strings.Fields(string(b))
|
||||
if len(f) < 3 {
|
||||
return 0, 0, 0
|
||||
}
|
||||
l1, _ = strconv.ParseFloat(f[0], 64)
|
||||
l5, _ = strconv.ParseFloat(f[1], 64)
|
||||
l15, _ = strconv.ParseFloat(f[2], 64)
|
||||
return l1, l5, l15
|
||||
}
|
||||
|
||||
// meminfo returns MemTotal and MemAvailable in bytes.
|
||||
func (s *LoadSampler) meminfo() (total, avail int64) {
|
||||
f, err := os.Open(filepath.Join(s.root, "meminfo"))
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
k, v, ok := strings.Cut(sc.Text(), ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(v)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
kb, _ := strconv.ParseInt(fields[0], 10, 64)
|
||||
switch k {
|
||||
case "MemTotal":
|
||||
total = kb * 1024
|
||||
case "MemAvailable":
|
||||
avail = kb * 1024
|
||||
}
|
||||
}
|
||||
return total, avail
|
||||
}
|
||||
|
||||
// procStat is what one /proc/<pid>/stat says that load needs.
|
||||
type procStat struct {
|
||||
pgrp int
|
||||
jiffies uint64
|
||||
rssPages int64
|
||||
}
|
||||
|
||||
// readProcStat parses /proc/<pid>/stat. The command name is in parentheses
|
||||
// and may itself hold spaces and parentheses, so fields are counted from the
|
||||
// last closing one.
|
||||
func readProcStat(path string) (procStat, bool) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return procStat{}, false
|
||||
}
|
||||
i := bytes.LastIndexByte(b, ')')
|
||||
if i < 0 {
|
||||
return procStat{}, false
|
||||
}
|
||||
f := strings.Fields(string(b[i+1:]))
|
||||
// After the name: state(0) ppid(1) pgrp(2) ... utime(11) stime(12) ... rss(21)
|
||||
if len(f) < 22 {
|
||||
return procStat{}, false
|
||||
}
|
||||
var st procStat
|
||||
st.pgrp, _ = strconv.Atoi(f[2])
|
||||
ut, _ := strconv.ParseUint(f[11], 10, 64)
|
||||
stt, _ := strconv.ParseUint(f[12], 10, 64)
|
||||
st.jiffies = ut + stt
|
||||
st.rssPages, _ = strconv.ParseInt(f[21], 10, 64)
|
||||
return st, true
|
||||
}
|
||||
|
||||
// groups totals CPU time and memory per process group, in one pass over
|
||||
// /proc. An engine is started as the leader of its own group, so its workers
|
||||
// — vLLM starts several — are counted with it.
|
||||
func (s *LoadSampler) groups() map[int]groupUse {
|
||||
out := map[int]groupUse{}
|
||||
s.eachProc(func(st procStat) {
|
||||
g := out[st.pgrp]
|
||||
g.jiffies += st.jiffies
|
||||
g.rssPages += st.rssPages
|
||||
out[st.pgrp] = g
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *LoadSampler) eachProc(fn func(procStat)) {
|
||||
entries, err := os.ReadDir(s.root)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if _, err := strconv.Atoi(e.Name()); err != nil {
|
||||
continue
|
||||
}
|
||||
if st, ok := readProcStat(filepath.Join(s.root, e.Name(), "stat")); ok {
|
||||
fn(st)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeProc is a /proc with two CPUs, a memory picture and processes.
|
||||
type fakeProc struct {
|
||||
t *testing.T
|
||||
root string
|
||||
}
|
||||
|
||||
func newFakeProc(t *testing.T) *fakeProc {
|
||||
f := &fakeProc{t: t, root: t.TempDir()}
|
||||
f.write("loadavg", "1.50 0.75 0.25 2/300 4242\n")
|
||||
f.write("meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 3000000 kB\n")
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeProc) write(name, body string) {
|
||||
f.t.Helper()
|
||||
p := filepath.Join(f.root, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// cpu sets the aggregate counters: user and idle, the rest zero.
|
||||
func (f *fakeProc) cpu(user, idle uint64) {
|
||||
f.write("stat", fmt.Sprintf("cpu %d 0 0 %d 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n", user, idle))
|
||||
}
|
||||
|
||||
// proc sets one process: its group, CPU jiffies (as utime) and RSS in pages.
|
||||
func (f *fakeProc) proc(pid, pgrp int, jiffies uint64, rss int64) {
|
||||
// The name holds a space and a parenthesis, as real ones can.
|
||||
f.write(fmt.Sprintf("%d/stat", pid), fmt.Sprintf(
|
||||
"%d (llama-server (x)) S 1 %d %d 0 -1 0 0 0 0 0 %d 0 0 0 20 0 1 0 100 1000 %d 0\n",
|
||||
pid, pgrp, pgrp, jiffies, rss))
|
||||
}
|
||||
|
||||
func TestLoadIsTheDifferenceBetweenTwoReadings(t *testing.T) {
|
||||
p := newFakeProc(t)
|
||||
at := time.Unix(1000, 0)
|
||||
s := newLoadSampler(p.root, func() time.Time { return at })
|
||||
|
||||
p.cpu(100, 900)
|
||||
p.proc(42, 42, 1000, 10)
|
||||
p.proc(43, 42, 500, 5) // a worker in the engine's group
|
||||
p.proc(7, 7, 99999, 1) // something else entirely
|
||||
if _, ok := s.Sample(map[string]int{"shard-a": 42}); ok {
|
||||
t.Fatal("the first reading has nothing to compare with and should report nothing")
|
||||
}
|
||||
|
||||
// Two seconds later: 300 of the 400 jiffies that passed were busy, and
|
||||
// the engine's group used 400 of them.
|
||||
at = at.Add(2 * time.Second)
|
||||
p.cpu(400, 1000)
|
||||
p.proc(42, 42, 1300, 10)
|
||||
p.proc(43, 42, 600, 6)
|
||||
l, ok := s.Sample(map[string]int{"shard-a": 42})
|
||||
if !ok {
|
||||
t.Fatal("the second reading should report")
|
||||
}
|
||||
if l.CPUPercent != 75 || l.CPUCount != 2 {
|
||||
t.Errorf("cpu %.1f%% of %d, want 75%% of 2", l.CPUPercent, l.CPUCount)
|
||||
}
|
||||
if l.Load1 != 1.5 || l.Load15 != 0.25 {
|
||||
t.Errorf("load %v %v", l.Load1, l.Load15)
|
||||
}
|
||||
if l.MemUsed != 5000000*1024 || l.MemAvail != 3000000*1024 {
|
||||
t.Errorf("memory used %d available %d", l.MemUsed, l.MemAvail)
|
||||
}
|
||||
if len(l.Shards) != 1 {
|
||||
t.Fatalf("shards = %+v", l.Shards)
|
||||
}
|
||||
sh := l.Shards[0]
|
||||
// 400 jiffies over 2 seconds at 100 a second is two cores.
|
||||
if sh.ID != "shard-a" || sh.CPUCores != 2 || sh.RSSBytes != 16*int64(os.Getpagesize()) {
|
||||
t.Errorf("shard = %+v", sh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoProcMeansNoLoad(t *testing.T) {
|
||||
s := newLoadSampler(filepath.Join(t.TempDir(), "missing"), time.Now)
|
||||
s.Sample(nil)
|
||||
if _, ok := s.Sample(nil); ok {
|
||||
t.Error("a system without /proc should report nothing, not zeros")
|
||||
}
|
||||
}
|
||||
@@ -207,6 +207,7 @@ func newServerCmd() *cobra.Command {
|
||||
web.NavExtras{History: history, Window: cfg.Monitoring.Window}, log))
|
||||
adminMux.Handle("GET /_static/", web.Static())
|
||||
adminMux.HandleFunc("GET /nodes/{id}/logs", web.NodeLogsPage(api, true, version.Version))
|
||||
adminMux.HandleFunc("GET /nodes/{id}", web.NodePage(api, history, cfg.Monitoring.Window, true, version.Version, log))
|
||||
|
||||
// The same page on the client-facing port at /admin, behind a
|
||||
// session and an admin flag. Handing the router the data rather
|
||||
|
||||
@@ -115,6 +115,43 @@ type NodeView struct {
|
||||
LastSeenAgo string `json:"last_seen_ago"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
Replicas int `json:"replicas"`
|
||||
|
||||
// Load is how busy the machine is, absent when its agent has not
|
||||
// measured it lately — which is unknown, not idle.
|
||||
Load *LoadView `json:"load,omitempty"`
|
||||
}
|
||||
|
||||
// LoadView is a node's CPU and memory in use, at its last heartbeat.
|
||||
type LoadView struct {
|
||||
CPUPercent float64 `json:"cpu_percent"`
|
||||
CPUs int `json:"cpus,omitempty"`
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
MemUsed int64 `json:"memory_used_bytes"`
|
||||
MemTotal int64 `json:"memory_total_bytes"`
|
||||
// MemPercent is used over used plus available.
|
||||
MemPercent float64 `json:"memory_percent"`
|
||||
AgeSec int64 `json:"age_seconds"`
|
||||
}
|
||||
|
||||
// loadFresh is how old a measurement may be and still be shown as current.
|
||||
const loadFresh = time.Minute
|
||||
|
||||
func loadView(l *inventory.Load, now time.Time) *LoadView {
|
||||
if l == nil || now.Sub(l.At) > loadFresh {
|
||||
return nil
|
||||
}
|
||||
v := &LoadView{
|
||||
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
|
||||
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
||||
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
|
||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||
}
|
||||
if v.MemTotal > 0 {
|
||||
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// ModelView is a model as `ctl models` shows it.
|
||||
@@ -170,6 +207,10 @@ type ShardView struct {
|
||||
// waiting on one. It is what tells a replica pulling a hundred gigabytes
|
||||
// apart from one that is wedged — which look identical without it.
|
||||
Fetch *FetchView `json:"fetch,omitempty"`
|
||||
// CPUCores and MemoryBytes are the engine's use of its node, from the
|
||||
// node's last heartbeat; zero when not measured.
|
||||
CPUCores float64 `json:"cpu_cores,omitempty"`
|
||||
MemoryBytes int64 `json:"memory_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// FetchView is a download in flight.
|
||||
@@ -266,6 +307,7 @@ func (a *API) Nodes() []NodeView {
|
||||
if !n.LastSeen.IsZero() {
|
||||
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
|
||||
}
|
||||
v.Load = loadView(n.Load, time.Now())
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
@@ -308,6 +350,12 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) {
|
||||
// Replicas builds the replica view, optionally filtered to one model.
|
||||
func (a *API) Replicas(model string) []ReplicaView {
|
||||
out := make([]ReplicaView, 0)
|
||||
loads := map[string]*inventory.Load{}
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
if n.Load != nil && time.Since(n.Load.At) <= loadFresh {
|
||||
loads[n.ID] = n.Load
|
||||
}
|
||||
}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
if model != "" && rep.Model != model {
|
||||
continue
|
||||
@@ -338,6 +386,11 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
sv.Fetch.Percent = &p
|
||||
}
|
||||
}
|
||||
if l := loads[s.NodeID]; l != nil {
|
||||
if use, ok := l.Shards[s.ID]; ok {
|
||||
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
|
||||
}
|
||||
}
|
||||
v.Shards = append(v.Shards, sv)
|
||||
}
|
||||
out = append(out, v)
|
||||
|
||||
@@ -188,3 +188,25 @@ func TestALogRequestIsAnsweredByTheNodeItWasSentTo(t *testing.T) {
|
||||
t.Error("a node with no stream answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeartbeatCarriesHowBusyTheNodeIs(t *testing.T) {
|
||||
s := newTestService()
|
||||
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready})
|
||||
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", Load: &llmgwv1.NodeLoad{
|
||||
CpuPercent: 62.5, CpuCount: 8, Load1: 3.2, MemoryUsedBytes: 6 << 30, MemoryAvailableBytes: 2 << 30,
|
||||
Shards: []*llmgwv1.ShardLoad{{ShardId: "r1-0", CpuCores: 4.5, MemoryRssBytes: 5 << 30}},
|
||||
}})
|
||||
n, _ := s.inv.Get("n1")
|
||||
if n.Load == nil || n.Load.CPUPercent != 62.5 || n.Load.MemUsedBytes != 6<<30 ||
|
||||
n.Load.Shards["r1-0"].CPUCores != 4.5 {
|
||||
t.Fatalf("load = %+v", n.Load)
|
||||
}
|
||||
|
||||
// A heartbeat that measured nothing — the agent's first — leaves the last
|
||||
// measurement standing rather than blanking it.
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
||||
if n, _ := s.inv.Get("n1"); n.Load == nil {
|
||||
t.Error("a heartbeat without load erased it")
|
||||
}
|
||||
}
|
||||
@@ -309,7 +309,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
memMB := int(hb.GetMemoryTotalMb())
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
|
||||
!hb.GetProvisionChanged() {
|
||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -400,9 +400,28 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
n.Provision = append(n.Provision, e)
|
||||
}
|
||||
}
|
||||
if l := hb.GetLoad(); l != nil {
|
||||
n.Load = loadFrom(l, time.Now())
|
||||
}
|
||||
s.inv.Upsert(n)
|
||||
}
|
||||
|
||||
// loadFrom is a heartbeat's load as the inventory keeps it.
|
||||
func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
|
||||
out := &inventory.Load{
|
||||
At: at, CPUPercent: l.GetCpuPercent(), CPUCount: int(l.GetCpuCount()),
|
||||
Load1: l.GetLoad1(), Load5: l.GetLoad5(), Load15: l.GetLoad15(),
|
||||
MemUsedBytes: l.GetMemoryUsedBytes(), MemAvailBytes: l.GetMemoryAvailableBytes(),
|
||||
Shards: map[string]inventory.ShardLoad{},
|
||||
}
|
||||
for _, sh := range l.GetShards() {
|
||||
out.Shards[sh.GetShardId()] = inventory.ShardLoad{
|
||||
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Service) attach(ctx context.Context, nodeID string) *conn {
|
||||
c := &conn{
|
||||
nodeID: nodeID,
|
||||
|
||||
@@ -59,8 +59,21 @@ type Collector struct {
|
||||
replicaState *prometheus.Desc
|
||||
replicaIdle *prometheus.Desc
|
||||
replicaReady *prometheus.Desc
|
||||
|
||||
nodeCPU *prometheus.Desc
|
||||
nodeCPUs *prometheus.Desc
|
||||
nodeLoad *prometheus.Desc
|
||||
nodeMemUsed *prometheus.Desc
|
||||
nodeMemAvail *prometheus.Desc
|
||||
shardCPU *prometheus.Desc
|
||||
shardMemory *prometheus.Desc
|
||||
}
|
||||
|
||||
// loadFresh is how old a node's load may be and still be reported. A node
|
||||
// that stopped sending it has an unknown load, and repeating its last figure
|
||||
// would draw a busy machine long after it went away.
|
||||
const loadFresh = time.Minute
|
||||
|
||||
// New builds the collector.
|
||||
func New(src Source) *Collector {
|
||||
node := []string{"node"}
|
||||
@@ -115,6 +128,25 @@ func New(src Source) *Collector {
|
||||
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
|
||||
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
|
||||
"which is not the same as idle.", []string{"model", "replica"}, nil),
|
||||
|
||||
// A node's own use, from its heartbeat. Absent when the agent has
|
||||
// not measured it lately, which is not the same as idle.
|
||||
nodeCPU: prometheus.NewDesc("llmgw_node_cpu_utilization_percent",
|
||||
"How busy a node's CPUs were between its last two heartbeats, 0 to 100 across all cores.", node, nil),
|
||||
nodeCPUs: prometheus.NewDesc("llmgw_node_cpus",
|
||||
"CPUs a node has, so a load average can be read against it.", node, nil),
|
||||
nodeLoad: prometheus.NewDesc("llmgw_node_load",
|
||||
"The kernel's load average over 1, 5 and 15 minutes.", append(node, "window"), nil),
|
||||
nodeMemUsed: prometheus.NewDesc("llmgw_node_memory_used_bytes",
|
||||
"RAM in use: total less what the kernel could hand to a new process.", node, nil),
|
||||
nodeMemAvail: prometheus.NewDesc("llmgw_node_memory_available_bytes",
|
||||
"RAM the kernel could hand to a new process without swapping.", node, nil),
|
||||
shardCPU: prometheus.NewDesc("llmgw_shard_cpu_cores",
|
||||
"Cores an engine process group kept busy, the way top counts: 2.5 is two and a half.",
|
||||
[]string{"node", "model", "replica", "shard"}, nil),
|
||||
shardMemory: prometheus.NewDesc("llmgw_shard_memory_bytes",
|
||||
"Resident memory of an engine process group.",
|
||||
[]string{"node", "model", "replica", "shard"}, nil),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +155,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
for _, d := range []*prometheus.Desc{
|
||||
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
|
||||
c.nodeDiskFree, c.nodeDiskSize,
|
||||
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
@@ -135,6 +168,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
now := time.Now()
|
||||
|
||||
loads := map[string]*inventory.Load{}
|
||||
for _, n := range c.src.Nodes() {
|
||||
fabric := ""
|
||||
if len(n.Fabrics) > 0 {
|
||||
@@ -165,6 +199,21 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
float64(total), n.ID)
|
||||
}
|
||||
|
||||
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
|
||||
if l.CPUCount > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
|
||||
}
|
||||
for w, v := range map[string]float64{"1m": l.Load1, "5m": l.Load5, "15m": l.Load15} {
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeLoad, prometheus.GaugeValue, v, n.ID, w)
|
||||
}
|
||||
if l.MemUsedBytes > 0 || l.MemAvailBytes > 0 {
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeMemUsed, prometheus.GaugeValue, float64(l.MemUsedBytes), n.ID)
|
||||
ch <- prometheus.MustNewConstMetric(c.nodeMemAvail, prometheus.GaugeValue, float64(l.MemAvailBytes), n.ID)
|
||||
}
|
||||
loads[n.ID] = l
|
||||
}
|
||||
|
||||
byClass := map[string]int{}
|
||||
for _, g := range n.GPUs {
|
||||
class := g.Class()
|
||||
@@ -207,6 +256,17 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
counts := map[[2]string]int{}
|
||||
ready := map[string]int{}
|
||||
for _, rep := range c.src.Replicas() {
|
||||
// Each engine's share of its node, labelled with what it serves, so
|
||||
// "which model is eating this machine" is one query.
|
||||
for _, sh := range rep.Shards {
|
||||
if l := loads[sh.NodeID]; l != nil {
|
||||
if use, ok := l.Shards[sh.ID]; ok {
|
||||
labels := []string{sh.NodeID, rep.Model, rep.ID, sh.ID}
|
||||
ch <- prometheus.MustNewConstMetric(c.shardCPU, prometheus.GaugeValue, use.CPUCores, labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.shardMemory, prometheus.GaugeValue, float64(use.RSSBytes), labels...)
|
||||
}
|
||||
}
|
||||
}
|
||||
counts[[2]string{rep.Model, rep.State.String()}]++
|
||||
if rep.State == reconciler.Ready {
|
||||
ready[rep.Model]++
|
||||
|
||||
@@ -139,3 +139,40 @@ func TestAnEmptyFleetCollectsWithoutError(t *testing.T) {
|
||||
t.Log("no metrics for an empty fleet, which is correct")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodesLoadAndEachEnginesShareAreReported(t *testing.T) {
|
||||
// The nodes are CPU machines: without this, how busy one was had no
|
||||
// answer anywhere.
|
||||
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(), Load: &inventory.Load{
|
||||
At: time.Now(), CPUPercent: 62.5, CPUCount: 8, Load1: 3.5, Load5: 2, Load15: 1,
|
||||
MemUsedBytes: 6 << 30, MemAvailBytes: 2 << 30,
|
||||
Shards: map[string]inventory.ShardLoad{"r1-0": {CPUCores: 4.5, RSSBytes: 5 << 30}},
|
||||
}}
|
||||
src := fakeSource{nodes: []inventory.Node{n}, replicas: []reconciler.Replica{{
|
||||
ID: "r1", Model: "qwen", State: reconciler.Ready,
|
||||
Shards: []reconciler.Shard{{ID: "r1-0", NodeID: "n1"}},
|
||||
}}}
|
||||
out := gather(t, src)
|
||||
for _, want := range []string{
|
||||
"llmgw_node_cpu_utilization_percent node=n1 -> 62.5",
|
||||
"llmgw_node_cpus node=n1 -> 8",
|
||||
"llmgw_node_load node=n1 window=1m -> 3.5",
|
||||
"llmgw_node_memory_used_bytes node=n1 -> 6442450944",
|
||||
"llmgw_shard_cpu_cores model=qwen node=n1 replica=r1 shard=r1-0 -> 4.5",
|
||||
"llmgw_shard_memory_bytes model=qwen node=n1 replica=r1 shard=r1-0 -> 5368709120",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStaleLoadIsAbsentNotRepeated(t *testing.T) {
|
||||
// A node that stopped reporting has an unknown load. Repeating its last
|
||||
// figure would draw a busy machine long after it went away.
|
||||
n := inventory.Node{ID: "n1", State: inventory.Ready, LastSeen: time.Now(),
|
||||
Load: &inventory.Load{At: time.Now().Add(-5 * time.Minute), CPUPercent: 90}}
|
||||
if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_node_cpu_utilization_percent") {
|
||||
t.Errorf("a stale load was reported:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,31 @@ type Node struct {
|
||||
Host discovery.Host `json:"host"`
|
||||
// Provision is where installing each wanted engine stands on this node.
|
||||
Provision []EngineInstall `json:"provision,omitempty"`
|
||||
|
||||
// Load is how busy the machine was at the last heartbeat that measured
|
||||
// it. Nil before the first one, and from an agent too old to say. Not
|
||||
// persisted: it describes a moment, and a restored one would be stale.
|
||||
Load *Load `json:"-"`
|
||||
}
|
||||
|
||||
// Load is a node's use of its CPU and memory, and each engine's share.
|
||||
type Load struct {
|
||||
At time.Time
|
||||
CPUPercent float64
|
||||
CPUCount int
|
||||
Load1 float64
|
||||
Load5 float64
|
||||
Load15 float64
|
||||
MemUsedBytes int64
|
||||
MemAvailBytes int64
|
||||
// Shards is each engine's use, by shard ID.
|
||||
Shards map[string]ShardLoad
|
||||
}
|
||||
|
||||
// ShardLoad is one engine's use: cores busy, and resident memory.
|
||||
type ShardLoad struct {
|
||||
CPUCores float64
|
||||
RSSBytes int64
|
||||
}
|
||||
|
||||
// EngineInstall is where one engine's installation stands on a node.
|
||||
|
||||
@@ -164,6 +164,8 @@ func (r *Router) Handler() http.Handler {
|
||||
}, r.log)))
|
||||
mux.HandleFunc("GET /admin/nodes/{id}/logs", pages.AdminPage(
|
||||
web.NodeLogsPage(r.clusterView, false, version.Version)))
|
||||
mux.HandleFunc("GET /admin/nodes/{id}", pages.AdminPage(
|
||||
web.NodePage(r.clusterView, r.history, r.histWindow, false, version.Version, r.log)))
|
||||
pages.RegisterAdminUsers(mux, r.store)
|
||||
// Quotas are written on the same tree the accounts page builds, so
|
||||
// they are mounted together.
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// nodePageData is one node: what it is, how busy it is now, and how busy it
|
||||
// has been.
|
||||
type nodePageData struct {
|
||||
Nav Nav
|
||||
OnAdminListener bool
|
||||
Node adminapi.NodeView
|
||||
Engines []nodeEngine
|
||||
Charts *nodeCharts
|
||||
HasHistory bool
|
||||
Back, Logs string
|
||||
// Self is this page's address, for the in-place refresh.
|
||||
Self string
|
||||
Version, Now string
|
||||
RefreshSec int
|
||||
}
|
||||
|
||||
// nodeEngine is one engine running on the node, and its share of it.
|
||||
type nodeEngine struct {
|
||||
Replica, Model, Shard, State string
|
||||
CPUCores float64
|
||||
MemoryBytes int64
|
||||
// CPUShare is its cores as a percentage of the machine, for the bar.
|
||||
CPUShare float64
|
||||
}
|
||||
|
||||
// nodeCharts are the node's recent history, from the metrics store.
|
||||
type nodeCharts struct {
|
||||
CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries
|
||||
Window string
|
||||
Err string
|
||||
}
|
||||
|
||||
// nodePanels are the charts, each a query scoped to this node. %s is the
|
||||
// node's ID, quoted.
|
||||
var nodePanels = []struct {
|
||||
expr, nameBy, unit string
|
||||
into func(*nodeCharts, TimeSeries)
|
||||
}{
|
||||
// Divided to a fraction because that is what the "%" unit draws.
|
||||
{`llmgw_node_cpu_utilization_percent{node=%s} / 100`, "node", "%",
|
||||
func(c *nodeCharts, t TimeSeries) { c.CPU = t }},
|
||||
{`llmgw_node_memory_used_bytes{node=%s}`, "node", "bytes",
|
||||
func(c *nodeCharts, t TimeSeries) { c.Memory = t }},
|
||||
{`llmgw_node_load{node=%s}`, "window", "",
|
||||
func(c *nodeCharts, t TimeSeries) { c.Load = t }},
|
||||
{`sum by (model) (llmgw_shard_cpu_cores{node=%s})`, "model", "",
|
||||
func(c *nodeCharts, t TimeSeries) { c.EngineCPU = t }},
|
||||
{`sum by (model) (llmgw_shard_memory_bytes{node=%s})`, "model", "bytes",
|
||||
func(c *nodeCharts, t TimeSeries) { c.EngineMemory = t }},
|
||||
}
|
||||
|
||||
// NodePage shows one node. Its charts come from the metrics store when one
|
||||
// is configured; the current figures come from the node's last heartbeat
|
||||
// either way.
|
||||
func NodePage(src Source, history History, window time.Duration, onAdminListener bool,
|
||||
version string, log *slog.Logger,
|
||||
) http.HandlerFunc {
|
||||
if window <= 0 {
|
||||
window = time.Hour
|
||||
}
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
d := nodePageData{
|
||||
Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener,
|
||||
Back: "/admin", Logs: "/admin/nodes/" + id + "/logs", Self: "/admin/nodes/" + id,
|
||||
HasHistory: history != nil, Version: version, RefreshSec: 15,
|
||||
Now: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if onAdminListener {
|
||||
d.Back, d.Logs, d.Self = "/", "/nodes/"+id+"/logs", "/nodes/"+id
|
||||
}
|
||||
found := false
|
||||
for _, n := range src.Nodes() {
|
||||
if n.ID == id {
|
||||
d.Node, found = n, true
|
||||
break
|
||||
}
|
||||
}
|
||||
// Only a node the control plane knows reaches a query below, which
|
||||
// is what keeps the ID a name rather than something typed into PromQL.
|
||||
if !found {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
var replicas []string
|
||||
for _, rep := range src.Replicas("") {
|
||||
for _, sh := range rep.Shards {
|
||||
if sh.Node != id {
|
||||
continue
|
||||
}
|
||||
e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State,
|
||||
CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes}
|
||||
if l := d.Node.Load; l != nil && l.CPUs > 0 {
|
||||
e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs)
|
||||
}
|
||||
d.Engines = append(d.Engines, e)
|
||||
replicas = append(replicas, rep.ID)
|
||||
}
|
||||
}
|
||||
sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores })
|
||||
|
||||
if history != nil {
|
||||
d.Charts = loadNodeCharts(r.Context(), history, id, replicas, window, log)
|
||||
}
|
||||
noStore(w)
|
||||
render(w, "node", d)
|
||||
}
|
||||
}
|
||||
|
||||
func loadNodeCharts(ctx context.Context, h History, node string, replicas []string,
|
||||
window time.Duration, log *slog.Logger,
|
||||
) *nodeCharts {
|
||||
step := window / 60
|
||||
out := &nodeCharts{Window: window.String()}
|
||||
quoted := fmt.Sprintf("%q", node)
|
||||
for _, p := range nodePanels {
|
||||
series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), p.nameBy, window, step)
|
||||
if err != nil {
|
||||
if log != nil {
|
||||
log.Warn("could not read a node's history", "node", node, "err", err)
|
||||
}
|
||||
out.Err = err.Error()
|
||||
return out
|
||||
}
|
||||
p.into(out, LineChart(series, p.unit, 4))
|
||||
}
|
||||
// The engines' queues, for the replicas that are here now.
|
||||
if len(replicas) > 0 {
|
||||
re := make([]string, len(replicas))
|
||||
for i, r := range replicas {
|
||||
re[i] = regexpQuote(r)
|
||||
}
|
||||
expr := fmt.Sprintf(`llmgw_endpoint_waiting{replica=~%q}`, strings.Join(re, "|"))
|
||||
if series, err := h.RangeQuery(ctx, expr, "replica", window, step); err == nil {
|
||||
out.Queue = LineChart(series, "", 4)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// regexpQuote escapes what a replica ID could carry into a PromQL regex.
|
||||
// IDs are letters, digits and dashes; this is for the day one is not.
|
||||
func regexpQuote(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if strings.ContainsRune(`\.+*?()|[]{}^$`, r) {
|
||||
b.WriteRune('\\')
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
|
||||
// nodeMux mounts the page where it lives, so the path carries the node.
|
||||
func nodeMux(src Source, h History) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
if h == nil {
|
||||
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, nil, time.Hour, false, "v1", nil))
|
||||
} else {
|
||||
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, h, time.Hour, false, "v1", nil))
|
||||
}
|
||||
return mux
|
||||
}
|
||||
|
||||
// busySource is fakeSource with a node that has been measured.
|
||||
type busySource struct{ fakeSource }
|
||||
|
||||
func (busySource) Nodes() []adminapi.NodeView {
|
||||
n := fakeSource{}.Nodes()[0]
|
||||
n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
|
||||
MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
|
||||
return []adminapi.NodeView{n}
|
||||
}
|
||||
|
||||
func (busySource) Replicas(string) []adminapi.ReplicaView {
|
||||
r := fakeSource{}.Replicas("")
|
||||
r[0].Shards[0].CPUCores, r[0].Shards[0].MemoryBytes = 4, 5<<30
|
||||
return r
|
||||
}
|
||||
|
||||
func TestANodesPageShowsHowBusyItIsAndWhatIsUsingIt(t *testing.T) {
|
||||
h := &fakeHistory{byExpr: map[string][]promq.Series{
|
||||
`llmgw_node_cpu_utilization_percent{node="llmgw-node01"} / 100`: {ts("llmgw-node01", 0.2, 0.6)},
|
||||
}}
|
||||
code, body := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"62%", "CPU · 8 cores", "RAM in use", "3.25",
|
||||
// The engine and its share: four of eight cores is half the machine.
|
||||
"<td>served</td>", "4.0 cores", "width: 50%",
|
||||
`hx-get="/admin/nodes/llmgw-node01"`,
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
// Every chart is about this node, and nothing else.
|
||||
for _, q := range h.asked {
|
||||
if !strings.Contains(q, `"llmgw-node01"`) && !strings.Contains(q, `served-1`) {
|
||||
t.Errorf("a query not scoped to the node: %s", q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownNodeIsNotFound(t *testing.T) {
|
||||
h := &fakeHistory{}
|
||||
code, _ := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/nope", nil)
|
||||
if code != http.StatusNotFound {
|
||||
t.Errorf("status %d", code)
|
||||
}
|
||||
// Nothing was asked of the metrics store for a name nobody knows.
|
||||
if len(h.asked) != 0 {
|
||||
t.Errorf("queried for an unknown node: %v", h.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
|
||||
_, body := get(t, nodeMux(fakeSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
||||
if !strings.Contains(body, "not measured lately") {
|
||||
t.Error("an unmeasured node should say so")
|
||||
}
|
||||
if !strings.Contains(body, "No metrics store is configured") {
|
||||
t.Error("without a metrics store the page should say where history comes from")
|
||||
}
|
||||
}
|
||||
@@ -534,3 +534,11 @@ table.repo-files td:last-child .opt { display: block; margin-bottom: .2rem; }
|
||||
.own-use .tile { min-width: 9rem; }
|
||||
/* Between what the cluster is and how it is being used, on the cluster page. */
|
||||
hr.part { border: 0; border-top: 1px solid var(--line); margin: 2.25rem 0 .5rem; }
|
||||
/* How full a node is, in the cluster page's node table and on its own page. */
|
||||
td.load { white-space: nowrap; min-width: 8.5rem; }
|
||||
.minibar { display: inline-block; width: 3.2rem; height: .45rem; border-radius: 999px; background: var(--bg);
|
||||
border: 1px solid var(--line); overflow: hidden; vertical-align: middle; margin-right: .4rem; }
|
||||
.minibar > span { display: block; height: 100%; background: var(--accent); }
|
||||
.minibar.warm > span { background: var(--warn); }
|
||||
.minibar.hot > span { background: var(--bad); }
|
||||
.load-v { font-variant-numeric: tabular-nums; font-size: .85rem; }
|
||||
@@ -51,13 +51,16 @@
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{if .Nodes}}
|
||||
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||||
<table class="nodes"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th><th class="num">GPUs</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||||
{{range .Nodes}}<tr>
|
||||
<td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}"><code>{{.ID}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
{{/* A dash, not an empty bar, when the agent has not measured it lately:
|
||||
unknown is not idle. */}}
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} <span class="note">of {{.CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">of {{bytes .MemTotal}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||||
<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}</td>
|
||||
{{/* Free out of total, because the first is meaningless without the
|
||||
second, and an em dash where nothing was reported: an agent that
|
||||
was never asked about its disk must not read as a full one. */}}
|
||||
@@ -68,9 +71,9 @@
|
||||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{.Replicas}}</td>
|
||||
<td>{{or .LastSeenAgo "—"}}</td>
|
||||
</tr>{{if .Unhealthy}}<tr><td colspan="11" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||||
{{range .Provision}}{{if ne .State "ready"}}<tr><td colspan="11" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||||
{{range .Warnings}}<tr><td colspan="11" class="note bad">{{.}}</td></tr>{{end}}
|
||||
</tr>{{if .Unhealthy}}<tr><td colspan="12" class="note bad">{{join .Unhealthy}}</td></tr>{{end}}
|
||||
{{range .Provision}}{{if ne .State "ready"}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||||
{{range .Warnings}}<tr><td colspan="12" class="note bad">{{.}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||||
|
||||
@@ -263,3 +266,6 @@ exposed:
|
||||
<p class="note">{{.note}}</p>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{/* A small horizontal bar for a percentage, coloured as it fills. */}}
|
||||
{{define "minibar"}}<span class="minibar{{if ge .P 90}} hot{{else if ge .P 75}} warm{{end}}"><span style="width: {{.P}}%"></span></span>{{end}}
|
||||
@@ -0,0 +1,69 @@
|
||||
{{define "node"}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<noscript><meta http-equiv="refresh" content="{{.RefreshSec}}"></noscript>
|
||||
<script src="/_static/htmx-2.0.11.min.js"></script>
|
||||
<title>llmgw — {{.Node.ID}}</title><link rel="stylesheet" href="{{stylesheet}}">
|
||||
</head><body><div class="wrap">
|
||||
<header>
|
||||
<h1>{{.Node.ID}}</h1>
|
||||
<p class="sub"><span class="state state-{{.Node.State}}">{{.Node.State}}</span>
|
||||
{{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}}
|
||||
· last seen {{or .Node.LastSeenAgo "never"}} ago · <a href="{{.Logs}}">logs</a> · <a href="{{.Back}}">cluster</a></p>
|
||||
{{if .OnAdminListener}}<nav><a href="{{.Back}}">cluster</a></nav>{{else}}{{template "nav" .Nav}}{{end}}
|
||||
</header>
|
||||
|
||||
{{/* Refreshed in place, like the cluster page. */}}
|
||||
<div id="live" hx-get="{{.Self}}" hx-trigger="every {{.RefreshSec}}s [document.visibilityState === 'visible']" hx-select="#live" hx-swap="outerHTML">
|
||||
{{with .Node}}
|
||||
<div class="tiles">
|
||||
{{with .Load}}
|
||||
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
|
||||
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
|
||||
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
|
||||
{{else}}
|
||||
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
|
||||
{{end}}
|
||||
{{if .DiskTotalGB}}<div class="tile"><div class="v">{{.DiskFreeGB}}G</div><div class="k">disk free · of {{.DiskTotalGB}}G</div></div>{{end}}
|
||||
{{if .GPUTotal}}<div class="tile"><div class="v">{{.GPUFree}}/{{.GPUTotal}}</div><div class="k">GPUs free</div></div>{{end}}
|
||||
</div>
|
||||
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
|
||||
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
|
||||
{{end}}
|
||||
|
||||
<h2>What runs here</h2>
|
||||
{{if .Engines}}
|
||||
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
||||
{{range .Engines}}<tr>
|
||||
<td>{{.Model}}</td>
|
||||
<td><code>{{.Replica}}</code></td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
<td class="load">{{template "minibar" pctbar .CPUShare}}<span class="load-v">{{printf "%.1f" .CPUCores}} cores</span></td>
|
||||
<td class="num">{{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{else}}<p class="empty">No engines are placed on this node.</p>{{end}}
|
||||
<p class="note">Engines: {{if .Node.Engines}}{{join .Node.Engines}}{{else}}none installed{{end}}{{if .Node.Models}} · {{len .Node.Models}} model file(s) on disk{{end}}.</p>
|
||||
</div>
|
||||
|
||||
{{if .HasHistory}}{{with .Charts}}
|
||||
<h2>The last {{.Window}}</h2>
|
||||
{{if .Err}}<p class="note error-row">No history: {{.Err}}</p>{{else}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .CPU "title" "CPU" "note" "All cores together, from the node's heartbeat."}}
|
||||
{{template "tschart" dict "t" .Memory "title" "RAM in use" "note" "Total less what the kernel could hand to a new process."}}
|
||||
</div>
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .EngineCPU "title" "Cores busy, by model" "note" "Each engine's process group, the way top counts it."}}
|
||||
{{template "tschart" dict "t" .EngineMemory "title" "Engine memory, by model" "note" "Resident memory: the weights and the KV cache, mostly."}}
|
||||
</div>
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Load "title" "Load average" "note" "Above the core count means work is waiting for a CPU."}}
|
||||
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started, on the replicas here now."}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}{{else}}
|
||||
<p class="note">No metrics store is configured, so there is no history here; <code>monitoring.prometheus</code> in the server configuration adds it.</p>
|
||||
{{end}}
|
||||
|
||||
<footer><span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span><span>{{.Now}}</span></footer>
|
||||
</div></body></html>{{end}}
|
||||
@@ -81,6 +81,11 @@ var funcs = template.FuncMap{
|
||||
return p
|
||||
},
|
||||
"ago": ago,
|
||||
// pctbar is a percentage as the minibar template takes it: whole, and
|
||||
// held to 0..100 so a measurement a little over does not draw past its box.
|
||||
"pctbar": func(v float64) struct{ P int } {
|
||||
return struct{ P int }{int(max(0, min(v, 100)) + 0.5)}
|
||||
},
|
||||
// Spend is stored in micro-units so accumulation is exact; it is shown in
|
||||
// whole units because that is the number an operator set a budget in.
|
||||
// compact renders a large count the way someone says it aloud: 1.2k rather
|
||||
|
||||
Reference in new issue
Block a user