diff --git a/CHANGELOG.md b/CHANGELOG.md index b19fa4b..7663057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,27 @@ decisions rather than only being printed: A model never measured behaves exactly as before. Measurements are read from the store every half minute. +### Nodes report their own heat and power + +The first step of facility-aware placement (docs/facility-aware-placement.md), +observe-only: nothing is decided on these yet. Every agent reads, at most every +ten seconds, and sends with its heartbeat: + +- **GPUs** from nvidia-smi: temperature, HBM temperature where the driver has + it, power draw and the enforced limit, and what is holding the clocks down — + power cap, thermal slowdown, power brake. A GPU's temperature was read once, + at registration; it is now current. +- **CPUs** from hwmon, which is what lm_sensors reads, with nothing to install: + the package temperature, and the board's inlet air sensor where it has one. +- **CPU power** from RAPL's energy counters. + +A sensor a machine does not have is not reported, never shown as zero. New +metrics: `llmgw_gpu_power_watts`, `llmgw_gpu_power_limit_watts`, +`llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_throttled{reason}`, +`llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts`, +`llmgw_node_inlet_temperature_celsius`. Each node's page shows them, with a GPU +table of temperature, power against its limit, and what is throttling it. + ### The mirror page in tabs /admin/mirror was one long page: what the mirror holds, the Hugging Face diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 41adf67..d2c353e 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20, 0} } // AgentMessage is anything a node tells the control plane. @@ -1415,8 +1415,15 @@ type NodeLoad struct { MemoryUsedBytes int64 `protobuf:"varint,6,opt,name=memory_used_bytes,json=memoryUsedBytes,proto3" json:"memory_used_bytes,omitempty"` MemoryAvailableBytes int64 `protobuf:"varint,7,opt,name=memory_available_bytes,json=memoryAvailableBytes,proto3" json:"memory_available_bytes,omitempty"` Shards []*ShardLoad `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // What the machine's own sensors say; zero is not measured. cpu_temp_c is + // the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL, + // inlet_temp_c the board's inlet or ambient sensor where it has one. + CpuTempC float64 `protobuf:"fixed64,9,opt,name=cpu_temp_c,json=cpuTempC,proto3" json:"cpu_temp_c,omitempty"` + CpuPowerW float64 `protobuf:"fixed64,10,opt,name=cpu_power_w,json=cpuPowerW,proto3" json:"cpu_power_w,omitempty"` + InletTempC float64 `protobuf:"fixed64,11,opt,name=inlet_temp_c,json=inletTempC,proto3" json:"inlet_temp_c,omitempty"` + Gpus []*GPUSensor `protobuf:"bytes,12,rep,name=gpus,proto3" json:"gpus,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeLoad) Reset() { @@ -1505,6 +1512,121 @@ func (x *NodeLoad) GetShards() []*ShardLoad { return nil } +func (x *NodeLoad) GetCpuTempC() float64 { + if x != nil { + return x.CpuTempC + } + return 0 +} + +func (x *NodeLoad) GetCpuPowerW() float64 { + if x != nil { + return x.CpuPowerW + } + return 0 +} + +func (x *NodeLoad) GetInletTempC() float64 { + if x != nil { + return x.InletTempC + } + return 0 +} + +func (x *NodeLoad) GetGpus() []*GPUSensor { + if x != nil { + return x.Gpus + } + return nil +} + +// GPUSensor is one card's heat, power, and what is holding it back. +type GPUSensor struct { + state protoimpl.MessageState `protogen:"open.v1"` + Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` + TempC float64 `protobuf:"fixed64,2,opt,name=temp_c,json=tempC,proto3" json:"temp_c,omitempty"` + MemoryTempC float64 `protobuf:"fixed64,3,opt,name=memory_temp_c,json=memoryTempC,proto3" json:"memory_temp_c,omitempty"` + PowerW float64 `protobuf:"fixed64,4,opt,name=power_w,json=powerW,proto3" json:"power_w,omitempty"` + PowerLimitW float64 `protobuf:"fixed64,5,opt,name=power_limit_w,json=powerLimitW,proto3" json:"power_limit_w,omitempty"` + // throttle names what is holding the clocks down now: sw_power_cap, + // hw_slowdown, sw_thermal, hw_thermal, power_brake. + Throttle []string `protobuf:"bytes,6,rep,name=throttle,proto3" json:"throttle,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GPUSensor) Reset() { + *x = GPUSensor{} + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GPUSensor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GPUSensor) ProtoMessage() {} + +func (x *GPUSensor) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GPUSensor.ProtoReflect.Descriptor instead. +func (*GPUSensor) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *GPUSensor) GetIndex() int32 { + if x != nil { + return x.Index + } + return 0 +} + +func (x *GPUSensor) GetTempC() float64 { + if x != nil { + return x.TempC + } + return 0 +} + +func (x *GPUSensor) GetMemoryTempC() float64 { + if x != nil { + return x.MemoryTempC + } + return 0 +} + +func (x *GPUSensor) GetPowerW() float64 { + if x != nil { + return x.PowerW + } + return 0 +} + +func (x *GPUSensor) GetPowerLimitW() float64 { + if x != nil { + return x.PowerLimitW + } + return 0 +} + +func (x *GPUSensor) GetThrottle() []string { + if x != nil { + return x.Throttle + } + return nil +} + // ShardLoad is one engine process group's use. type ShardLoad struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1520,7 +1642,7 @@ type ShardLoad struct { func (x *ShardLoad) Reset() { *x = ShardLoad{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1532,7 +1654,7 @@ func (x *ShardLoad) String() string { func (*ShardLoad) ProtoMessage() {} func (x *ShardLoad) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1545,7 +1667,7 @@ func (x *ShardLoad) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead. func (*ShardLoad) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *ShardLoad) GetShardId() string { @@ -1613,7 +1735,7 @@ type IBPort struct { func (x *IBPort) Reset() { *x = IBPort{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1625,7 +1747,7 @@ func (x *IBPort) String() string { func (*IBPort) ProtoMessage() {} func (x *IBPort) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1638,7 +1760,7 @@ func (x *IBPort) ProtoReflect() protoreflect.Message { // Deprecated: Use IBPort.ProtoReflect.Descriptor instead. func (*IBPort) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *IBPort) GetHca() string { @@ -1781,7 +1903,7 @@ type SubnetManager struct { func (x *SubnetManager) Reset() { *x = SubnetManager{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1793,7 +1915,7 @@ func (x *SubnetManager) String() string { func (*SubnetManager) ProtoMessage() {} func (x *SubnetManager) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1806,7 +1928,7 @@ func (x *SubnetManager) ProtoReflect() protoreflect.Message { // Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead. func (*SubnetManager) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *SubnetManager) GetGuid() string { @@ -1849,7 +1971,7 @@ type IMEX struct { func (x *IMEX) Reset() { *x = IMEX{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1861,7 +1983,7 @@ func (x *IMEX) String() string { func (*IMEX) ProtoMessage() {} func (x *IMEX) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1874,7 +1996,7 @@ func (x *IMEX) ProtoReflect() protoreflect.Message { // Deprecated: Use IMEX.ProtoReflect.Descriptor instead. func (*IMEX) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *IMEX) GetState() string { @@ -1906,7 +2028,7 @@ type GPUFabricState struct { func (x *GPUFabricState) Reset() { *x = GPUFabricState{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1918,7 +2040,7 @@ func (x *GPUFabricState) String() string { func (*GPUFabricState) ProtoMessage() {} func (x *GPUFabricState) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1931,7 +2053,7 @@ func (x *GPUFabricState) ProtoReflect() protoreflect.Message { // Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead. func (*GPUFabricState) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *GPUFabricState) GetIndex() int32 { @@ -2038,7 +2160,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2050,7 +2172,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2063,7 +2185,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *Heartbeat) GetNodeId() string { @@ -2211,7 +2333,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2223,7 +2345,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2236,7 +2358,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *Provision) GetEngines() []*EngineWant { @@ -2257,7 +2379,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2269,7 +2391,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2282,7 +2404,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *EngineWant) GetEngine() string { @@ -2315,7 +2437,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2327,7 +2449,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2340,7 +2462,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *ProvisionStatus) GetEngine() string { @@ -2404,7 +2526,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2416,7 +2538,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2429,7 +2551,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *ShardStatus) GetNodeId() string { @@ -2503,7 +2625,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2515,7 +2637,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2528,7 +2650,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *FetchProgress) GetSource() string { @@ -2591,7 +2713,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2603,7 +2725,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2616,7 +2738,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *StartShard) GetReplicaId() string { @@ -2757,7 +2879,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2769,7 +2891,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2782,7 +2904,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *EngineProfile) GetName() string { @@ -2878,7 +3000,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2890,7 +3012,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2903,7 +3025,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *RemoveWeights) GetPaths() []string { @@ -2923,7 +3045,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2935,7 +3057,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2948,7 +3070,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *StopShard) GetShardId() string { @@ -2968,7 +3090,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2980,7 +3102,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2993,7 +3115,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} } func (x *DrainShard) GetShardId() string { @@ -3015,7 +3137,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3027,7 +3149,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[26] + mi := &file_llmgw_v1_agent_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3040,7 +3162,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} } func (x *Ack) GetMessageId() string { @@ -3084,7 +3206,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3096,7 +3218,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[27] + mi := &file_llmgw_v1_agent_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3109,7 +3231,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} } func (x *LogsRequest) GetRequestId() string { @@ -3149,7 +3271,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3161,7 +3283,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[28] + mi := &file_llmgw_v1_agent_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3174,7 +3296,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} } func (x *LogsReply) GetRequestId() string { @@ -3220,7 +3342,7 @@ type FabricScanRequest struct { func (x *FabricScanRequest) Reset() { *x = FabricScanRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3232,7 +3354,7 @@ func (x *FabricScanRequest) String() string { func (*FabricScanRequest) ProtoMessage() {} func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[29] + mi := &file_llmgw_v1_agent_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3245,7 +3367,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead. func (*FabricScanRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} } func (x *FabricScanRequest) GetRequestId() string { @@ -3284,7 +3406,7 @@ type FabricScanReply struct { func (x *FabricScanReply) Reset() { *x = FabricScanReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3296,7 +3418,7 @@ func (x *FabricScanReply) String() string { func (*FabricScanReply) ProtoMessage() {} func (x *FabricScanReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[30] + mi := &file_llmgw_v1_agent_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3309,7 +3431,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead. func (*FabricScanReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} } func (x *FabricScanReply) GetRequestId() string { @@ -3365,7 +3487,7 @@ type FabricTestRequest struct { func (x *FabricTestRequest) Reset() { *x = FabricTestRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3377,7 +3499,7 @@ func (x *FabricTestRequest) String() string { func (*FabricTestRequest) ProtoMessage() {} func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[31] + mi := &file_llmgw_v1_agent_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3390,7 +3512,7 @@ func (x *FabricTestRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead. func (*FabricTestRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} } func (x *FabricTestRequest) GetRequestId() string { @@ -3459,7 +3581,7 @@ type FabricTestReply struct { func (x *FabricTestReply) Reset() { *x = FabricTestReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3471,7 +3593,7 @@ func (x *FabricTestReply) String() string { func (*FabricTestReply) ProtoMessage() {} func (x *FabricTestReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[32] + mi := &file_llmgw_v1_agent_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3484,7 +3606,7 @@ func (x *FabricTestReply) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead. func (*FabricTestReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} } func (x *FabricTestReply) GetRequestId() string { @@ -3544,7 +3666,7 @@ type UpgradeAgent struct { func (x *UpgradeAgent) Reset() { *x = UpgradeAgent{} - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3556,7 +3678,7 @@ func (x *UpgradeAgent) String() string { func (*UpgradeAgent) ProtoMessage() {} func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[33] + mi := &file_llmgw_v1_agent_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3569,7 +3691,7 @@ func (x *UpgradeAgent) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead. func (*UpgradeAgent) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} } func (x *UpgradeAgent) GetRequestId() string { @@ -3606,7 +3728,7 @@ type UpgradeReply struct { func (x *UpgradeReply) Reset() { *x = UpgradeReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3618,7 +3740,7 @@ func (x *UpgradeReply) String() string { func (*UpgradeReply) ProtoMessage() {} func (x *UpgradeReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[34] + mi := &file_llmgw_v1_agent_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3631,7 +3753,7 @@ func (x *UpgradeReply) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead. func (*UpgradeReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35} } func (x *UpgradeReply) GetRequestId() string { @@ -3786,7 +3908,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" + - "\x06shared\x18\x03 \x01(\bR\x06shared\"\x9b\x02\n" + + "\x06shared\x18\x03 \x01(\bR\x06shared\"\xa4\x03\n" + "\bNodeLoad\x12\x1f\n" + "\vcpu_percent\x18\x01 \x01(\x01R\n" + "cpuPercent\x12\x1b\n" + @@ -3796,7 +3918,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06load15\x18\x05 \x01(\x01R\x06load15\x12*\n" + "\x11memory_used_bytes\x18\x06 \x01(\x03R\x0fmemoryUsedBytes\x124\n" + "\x16memory_available_bytes\x18\a \x01(\x03R\x14memoryAvailableBytes\x12+\n" + - "\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\"m\n" + + "\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\x12\x1c\n" + + "\n" + + "cpu_temp_c\x18\t \x01(\x01R\bcpuTempC\x12\x1e\n" + + "\vcpu_power_w\x18\n" + + " \x01(\x01R\tcpuPowerW\x12 \n" + + "\finlet_temp_c\x18\v \x01(\x01R\n" + + "inletTempC\x12'\n" + + "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xb5\x01\n" + + "\tGPUSensor\x12\x14\n" + + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" + + "\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" + + "\rmemory_temp_c\x18\x03 \x01(\x01R\vmemoryTempC\x12\x17\n" + + "\apower_w\x18\x04 \x01(\x01R\x06powerW\x12\"\n" + + "\rpower_limit_w\x18\x05 \x01(\x01R\vpowerLimitW\x12\x1a\n" + + "\bthrottle\x18\x06 \x03(\tR\bthrottle\"m\n" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + @@ -4026,7 +4162,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 41) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -4039,91 +4175,93 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad - (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad - (*IBPort)(nil), // 12: llmgw.v1.IBPort - (*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager - (*IMEX)(nil), // 14: llmgw.v1.IMEX - (*GPUFabricState)(nil), // 15: llmgw.v1.GPUFabricState - (*Heartbeat)(nil), // 16: llmgw.v1.Heartbeat - (*Provision)(nil), // 17: llmgw.v1.Provision - (*EngineWant)(nil), // 18: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 19: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 20: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 21: llmgw.v1.FetchProgress - (*StartShard)(nil), // 22: llmgw.v1.StartShard - (*EngineProfile)(nil), // 23: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 24: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 25: llmgw.v1.StopShard - (*DrainShard)(nil), // 26: llmgw.v1.DrainShard - (*Ack)(nil), // 27: llmgw.v1.Ack - (*LogsRequest)(nil), // 28: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 29: llmgw.v1.LogsReply - (*FabricScanRequest)(nil), // 30: llmgw.v1.FabricScanRequest - (*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply - (*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest - (*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply - (*UpgradeAgent)(nil), // 34: llmgw.v1.UpgradeAgent - (*UpgradeReply)(nil), // 35: llmgw.v1.UpgradeReply - nil, // 36: llmgw.v1.Register.LabelsEntry - nil, // 37: llmgw.v1.Register.EngineVersionsEntry - nil, // 38: llmgw.v1.GPU.NicPathsEntry - nil, // 39: llmgw.v1.IBPort.CountersEntry - nil, // 40: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 41: llmgw.v1.EngineProfile.EnvEntry + (*GPUSensor)(nil), // 11: llmgw.v1.GPUSensor + (*ShardLoad)(nil), // 12: llmgw.v1.ShardLoad + (*IBPort)(nil), // 13: llmgw.v1.IBPort + (*SubnetManager)(nil), // 14: llmgw.v1.SubnetManager + (*IMEX)(nil), // 15: llmgw.v1.IMEX + (*GPUFabricState)(nil), // 16: llmgw.v1.GPUFabricState + (*Heartbeat)(nil), // 17: llmgw.v1.Heartbeat + (*Provision)(nil), // 18: llmgw.v1.Provision + (*EngineWant)(nil), // 19: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 20: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 21: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 22: llmgw.v1.FetchProgress + (*StartShard)(nil), // 23: llmgw.v1.StartShard + (*EngineProfile)(nil), // 24: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 25: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 26: llmgw.v1.StopShard + (*DrainShard)(nil), // 27: llmgw.v1.DrainShard + (*Ack)(nil), // 28: llmgw.v1.Ack + (*LogsRequest)(nil), // 29: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 30: llmgw.v1.LogsReply + (*FabricScanRequest)(nil), // 31: llmgw.v1.FabricScanRequest + (*FabricScanReply)(nil), // 32: llmgw.v1.FabricScanReply + (*FabricTestRequest)(nil), // 33: llmgw.v1.FabricTestRequest + (*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply + (*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent + (*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply + nil, // 37: llmgw.v1.Register.LabelsEntry + nil, // 38: llmgw.v1.Register.EngineVersionsEntry + nil, // 39: llmgw.v1.GPU.NicPathsEntry + nil, // 40: llmgw.v1.IBPort.CountersEntry + nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 42: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 16, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 20, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply - 33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply - 35, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply - 22, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 25, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 26, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 27, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 24, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 17, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 28, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 30, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest - 32, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest - 34, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent - 36, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 17, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 21, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 30, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 32, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply + 34, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply + 36, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply + 23, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 26, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 27, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 28, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 25, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 18, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 29, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest + 33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest + 35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent + 37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 37, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host - 14, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX + 15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX 0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 38, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry - 11, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 39, // 29: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 13, // 30: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager - 5, // 31: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 32: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 40, // 33: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 19, // 34: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 35: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 12, // 36: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 8, // 37: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 15, // 38: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState - 14, // 39: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX - 18, // 40: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 41: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 21, // 42: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 23, // 43: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 41, // 44: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 45: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 46: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 46, // [46:47] is the sub-list for method output_type - 45, // [45:46] is the sub-list for method input_type - 45, // [45:45] is the sub-list for extension type_name - 45, // [45:45] is the sub-list for extension extendee - 0, // [0:45] is the sub-list for field type_name + 39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad + 11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor + 40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 8, // 38: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 16, // 39: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState + 15, // 40: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX + 19, // 41: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 47, // [47:48] is the sub-list for method output_type + 46, // [46:47] is the sub-list for method input_type + 46, // [46:46] is the sub-list for extension type_name + 46, // [46:46] is the sub-list for extension extendee + 0, // [0:46] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -4158,7 +4296,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 41, + NumMessages: 42, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index cd24d04..3283981 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -285,6 +285,25 @@ message NodeLoad { int64 memory_used_bytes = 6; int64 memory_available_bytes = 7; repeated ShardLoad shards = 8; + // What the machine's own sensors say; zero is not measured. cpu_temp_c is + // the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL, + // inlet_temp_c the board's inlet or ambient sensor where it has one. + double cpu_temp_c = 9; + double cpu_power_w = 10; + double inlet_temp_c = 11; + repeated GPUSensor gpus = 12; +} + +// GPUSensor is one card's heat, power, and what is holding it back. +message GPUSensor { + int32 index = 1; + double temp_c = 2; + double memory_temp_c = 3; + double power_w = 4; + double power_limit_w = 5; + // throttle names what is holding the clocks down now: sw_power_cap, + // hw_slowdown, sw_thermal, hw_thermal, power_brake. + repeated string throttle = 6; } // ShardLoad is one engine process group's use. diff --git a/docs/facility-aware-placement.md b/docs/facility-aware-placement.md index 93388e0..953eb85 100644 --- a/docs/facility-aware-placement.md +++ b/docs/facility-aware-placement.md @@ -1,9 +1,11 @@ # Facility-aware placement: rack power, temperature and cooling -**Status: planned, not built.** This is the design for step 5 of resource -allocation, after reserved CPU cores and memory (1), capacity classes and -preemption (2), tenant capacity quotas (3) and sizing by measured throughput -(4), all of which shipped in v0.10.5. Nothing here exists in the code yet. +**Status: planned; its first step, reading the nodes' own sensors, is built.** +This is the design for step 5 of resource allocation, after reserved CPU cores +and memory (1), capacity classes and preemption (2), tenant capacity quotas (3) +and sizing by measured throughput (4), all of which shipped in v0.10.5. Of +what follows, only the node sensors (rollout step 0) exist in the code: they +are read, published and shown, and change no decision. ## Why @@ -35,7 +37,7 @@ and never let missing data stop the cluster. | Headroom: the scheduler dry-run on a copy of the inventory | Room that respects power budgets, for free | | Preemption with classes, minimums and a dry run (step 2) | Shedding load from a hot rack in the right order | | Measured throughput per profile (step 4) | Tokens per watt as a placement preference | -| GPU temperature from NVML, per card | A node-level thermal signal that needs no facility data | +| Node sensors, read by every agent (step 0, built) | Heat and power with no facility integration: see below | | `internal/promq`, a Prometheus client | Reading facility metrics that DCIM exporters publish | ## Concepts @@ -116,8 +118,8 @@ starts nor ends anything: | Condition | Enters when | Placement | |---|---|---| | `ok` | below the warm threshold for 5 min | normal | -| `warm` | inlet above the warm threshold for 2 min, or GPU temperatures near throttling | allowed, scored down: other domains first | -| `hot` | inlet above the hot threshold for 2 min, or a cooling unit in alarm | no new placements, as a cordon would, but automatic and reversible | +| `warm` | inlet above the warm threshold for 2 min, GPU temperatures near throttling, or any GPU thermally throttled | allowed, scored down: other domains first | +| `hot` | inlet above the hot threshold for 2 min, a cooling unit in alarm, or a sustained share of the domain's GPUs thermally throttled | no new placements, as a cordon would, but automatic and reversible | | `unknown` | no fresh reading | as `ok`, and reported as unknown | **Shedding** (optional, off by default): a domain `hot` for longer than its @@ -186,9 +188,39 @@ type FacilitySource interface { query: max by (row) (crac_alarm) domain_label: row ``` -- **Agents.** What the nodes can read themselves: GPU power and temperature from - NVML, node power from Redfish or IPMI where the BMC allows it. Needs no - facility integration, and gives node-level signals on day one. +- **Agents (built, step 0).** What each node reads about itself, every ten + seconds at most, sent with its heartbeat: + - **GPUs, from nvidia-smi:** temperature, memory (HBM) temperature where the + driver reports it, power draw, the enforced power limit, and what is + holding the clocks down now — `sw_power_cap`, `hw_slowdown`, `sw_thermal`, + `hw_thermal`, `power_brake`. + - **CPUs, from hwmon** (what lm_sensors reads, without it installed): the + package temperature, or the hottest core; and the board's inlet or ambient + sensor where it has one (`SYSTIN`, `Inlet`, `Ambient`). + - **CPU power, from RAPL:** the packages' energy counters over time, + subdomains not counted twice, counter wrap handled. + + A machine without a sensor reports nothing for it, never a zero; a VM has + none of them. Published as `llmgw_gpu_temperature_celsius`, + `llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_power_watts`, + `llmgw_gpu_power_limit_watts`, `llmgw_gpu_throttled{reason}`, + `llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts` and + `llmgw_node_inlet_temperature_celsius`, and on each node's page. + + What they can and cannot say: + - **Throttling is the strongest signal there is.** A temperature is a proxy; + a card holding its clocks down is the fact that invalidates its measured + capacity, and it needs no threshold to be chosen. + - **Measured power** is GPU draw plus CPU packages; memory, disks, fans and + power-supply losses are not in it, so a node's draw is that plus an + estimated remainder. A rack's is its llmgw nodes' plus a declared figure + for everything else in it: switches and storage report nothing here. + - **CPU and GPU temperatures follow the load, not the room.** A hot rack is + inferred from many nodes running hot at comparable load, or from + throttling, not from one node's CPU. Only a board inlet sensor reads the + air, and most boards have none. + - **No budget comes from a sensor.** A reading is a draw; the cap it is + measured against still comes from the facility. - **Simulated.** Readings from a script, for tests and for rehearsing a thermal event on a cluster without facility data. @@ -223,14 +255,17 @@ type FacilitySource interface { Each step ships on its own and is useful without the next: +0. **Node sensors, observe-only (built).** GPU heat, power and throttling, + CPU heat and power, inlet air where the board has a sensor; published and + shown, no decision changed. 1. **Location and spreading.** Well-known labels, domains from labels, replicas spread across racks and PDUs, locations on the node views. No facility data needed. 2. **Power budgets from a static file.** Caps and margins, estimated draw per replica (declared or derived), hard limit in placement and headroom. -3. **Readings, observe-only.** The Prometheus and agent sources, conditions - with hysteresis, measured power folded into headroom; decisions shown, not - enforced. +3. **Readings, observe-only.** The Prometheus source beside the agents', + conditions with hysteresis (throttling first among them), measured power + folded into headroom; decisions shown, not enforced. 4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and release. 5. **Shedding and tokens per watt.** Opt-in shedding through step 2's diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 72afbf3..e5d6993 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -101,6 +101,9 @@ type Agent struct { smsErr map[string]string // load measures the machine between heartbeats. load *discovery.LoadSampler + // sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs + // from nvidia-smi. + sensors *discovery.SensorSampler // fetchWatchers are the shards waiting on each source, so progress on one // download reaches every shard that asked for it. A fetch is single-flight @@ -146,6 +149,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er logRing: ring, provKick: make(chan struct{}, 1), load: discovery.NewLoadSampler(), + sensors: discovery.NewSensorSampler(), } a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus} return a, nil @@ -638,6 +642,16 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad { ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes, }) } + if a.sensors != nil { + se := a.sensors.Sample(context.Background()) + out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC + for _, g := range se.GPUs { + out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{ + Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC, + PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle, + }) + } + } return out } diff --git a/internal/agent/discovery/sensors.go b/internal/agent/discovery/sensors.go new file mode 100644 index 0000000..d415e31 --- /dev/null +++ b/internal/agent/discovery/sensors.go @@ -0,0 +1,242 @@ +package discovery + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// Sensors is what a node's own hardware says about heat and power: the CPUs, +// the board where it has an inlet sensor, and each GPU. A zero is not +// measured, never a reading of zero — a VM has none of these, and must not +// look like a machine running at 0 °C on no power. +type Sensors struct { + // CPUTempC is the hottest CPU sensor: the package, or the hottest core + // where no package reading exists. + CPUTempC float64 + // CPUPowerW is the CPU packages' draw, from RAPL: the change in their + // energy counters over the time between two samples. + CPUPowerW float64 + // InletTempC is the air coming in, where the board has a sensor it calls + // that. Most do not, and a CPU's temperature is not a substitute: it + // follows the load, not the room. + InletTempC float64 + GPUs []GPUSensor +} + +// GPUSensor is one card's heat, power and whether it is slowing itself down. +type GPUSensor struct { + Index int + TempC float64 + MemoryTempC float64 + PowerW float64 + PowerLimitW float64 + // Throttle names what is holding the card's clocks down now: + // sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty + // is running free. Idle and application clock settings are left out; + // they are not the card being held back. + Throttle []string +} + +// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often +// than temperatures change, and each run costs a few hundred milliseconds. +const sensorEvery = 10 * time.Second + +// SensorSampler reads Sensors. The first sample has no CPU power, which needs +// two readings of the energy counters. +type SensorSampler struct { + sys string // /sys, or a test's copy of it + smi func(ctx context.Context, fields string) ([]byte, error) + now func() time.Time + + mu sync.Mutex + last Sensors + lastAt time.Time + prevEnergy map[string]uint64 + prevAt time.Time +} + +// NewSensorSampler reads this machine. +func NewSensorSampler() *SensorSampler { + return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now} +} + +func runSMI(ctx context.Context, fields string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + //nolint:gosec // fixed binary, fields from this file + return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields, + "--format=csv,noheader,nounits").Output() +} + +// Sample reads the sensors, or returns the last reading when that is recent. +func (s *SensorSampler) Sample(ctx context.Context) Sensors { + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery { + return s.last + } + out := Sensors{} + out.CPUTempC, out.InletTempC = s.hwmon() + out.CPUPowerW = s.rapl(now) + out.GPUs = s.gpus(ctx) + s.last, s.lastAt = out, now + return out +} + +// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon +// device the kernel exposes — what lm_sensors reads, without it installed. +func (s *SensorSampler) hwmon() (cpu, inlet float64) { + devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*")) + var pkg, core float64 + for _, d := range devs { + name := readTrim(filepath.Join(d, "name")) + inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input")) + for _, in := range inputs { + milli, err := strconv.ParseFloat(readTrim(in), 64) + if err != nil || milli <= 0 { + continue + } + c := milli / 1000 + label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label")) + switch { + case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"): + pkg = max(pkg, c) + case isCPUSensor(name): + core = max(core, c) + case label == "cpu": + // A board's own reading of the CPU, as Dell's SMM gives. + core = max(core, c) + case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") || + strings.Contains(label, "intake") || label == "systin": + inlet = max(inlet, c) + } + } + } + if pkg > 0 { + return pkg, inlet + } + return core, inlet +} + +func isCPUSensor(name string) bool { + switch name { + case "coretemp", "k10temp", "zenpower", "cpu_thermal": + return true + } + return false +} + +// rapl is the CPU packages' power from their energy counters. Package +// domains only: the dram and core subdomains are parts of a package, and +// adding them would count the same energy twice. +func (s *SensorSampler) rapl(now time.Time) float64 { + doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*")) + energy := map[string]uint64{} + wrap := map[string]uint64{} + for _, d := range doms { + if strings.Count(filepath.Base(d), ":") != 1 { + continue // intel-rapl:0:0 is a subdomain of intel-rapl:0 + } + e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64) + if err != nil { + continue // unreadable without root on newer kernels + } + energy[d] = e + wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64) + } + prev, prevAt := s.prevEnergy, s.prevAt + s.prevEnergy, s.prevAt = energy, now + dt := now.Sub(prevAt).Seconds() + if prev == nil || dt <= 0 || len(energy) == 0 { + return 0 + } + var joules float64 + for d, e := range energy { + p, ok := prev[d] + if !ok { + return 0 // a domain appeared: no fair difference this time + } + if e < p { // the counter wrapped + e += wrap[d] + } + joules += float64(e-p) / 1e6 + } + return joules / dt +} + +// The throttle reasons nvidia-smi reports, as bits. Only those that mean the +// card is being held back are named. +var throttleBits = []struct { + bit uint64 + name string +}{ + {0x4, "sw_power_cap"}, + {0x8, "hw_slowdown"}, + {0x20, "sw_thermal"}, + {0x40, "hw_thermal"}, + {0x80, "power_brake"}, +} + +// gpus asks nvidia-smi for each card's heat and power. The memory temperature +// is not known to every driver, and a field nvidia-smi does not know fails the +// whole query, so it is asked for first and dropped on failure. +func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor { + withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory" + out, err := s.smi(ctx, withMem) + if err != nil { + if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil { + return nil + } + } + var gs []GPUSensor + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + f := strings.Split(line, ",") + if len(f) < 5 { + continue + } + for i := range f { + f[i] = strings.TrimSpace(f[i]) + } + idx, err := strconv.Atoi(f[0]) + if err != nil { + continue + } + g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])} + if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil { + for _, b := range throttleBits { + if bits&b.bit != 0 { + g.Throttle = append(g.Throttle, b.name) + } + } + } + if len(f) > 5 { + g.MemoryTempC = num(f[5]) + } + gs = append(gs, g) + } + return gs +} + +// num reads a figure nvidia-smi may give as "[N/A]" or "N/A". +func num(s string) float64 { + v, err := strconv.ParseFloat(s, 64) + if err != nil || v < 0 { + return 0 + } + return v +} + +func readTrim(path string) string { + b, err := os.ReadFile(path) //nolint:gosec // sysfs + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} diff --git a/internal/agent/discovery/sensors_test.go b/internal/agent/discovery/sensors_test.go new file mode 100644 index 0000000..ce1507e --- /dev/null +++ b/internal/agent/discovery/sensors_test.go @@ -0,0 +1,169 @@ +package discovery + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func put(t *testing.T, root, rel, body string) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil { + t.Fatal(err) + } +} + +// dellHost is the layout of the Proxmox host these tests were written +// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package +// and cores, and RAPL with a package and its dram subdomain. +func dellHost(t *testing.T) string { + root := t.TempDir() + put(t, root, "class/hwmon/hwmon0/name", "dell_smm") + put(t, root, "class/hwmon/hwmon0/temp1_input", "55000") + put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU") + put(t, root, "class/hwmon/hwmon0/temp2_input", "47000") + put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM") + put(t, root, "class/hwmon/hwmon1/name", "coretemp") + put(t, root, "class/hwmon/hwmon1/temp1_input", "58000") + put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0") + put(t, root, "class/hwmon/hwmon1/temp10_input", "61000") + put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8") + put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000") + put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850") + put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000") + return root +} + +func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler { + return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }} +} + +func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") } + +func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) { + root := dellHost(t) + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := sampler(root, noSMI, &now) + + first := s.Sample(t.Context()) + // The package reading, not the hotter core: the package is what the + // chip itself reports as its temperature. + if first.CPUTempC != 58 { + t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC) + } + if first.CPUPowerW != 0 { + t.Error("power from one reading of a counter is invented") + } + if first.InletTempC != 0 || first.GPUs != nil { + t.Errorf("sensors this host does not have were reported: %+v", first) + } + + // 10 s later the package has used 650 J, and its dram subdomain some of + // that — counted once. + put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000") + put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000") + now = now.Add(10 * time.Second) + if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 { + t.Errorf("CPU power %.2f W, expected 65", got) + } +} + +func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) { + root := dellHost(t) + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000") + s := sampler(root, noSMI, &now) + s.Sample(t.Context()) + // 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s. + put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150") + now = now.Add(10 * time.Second) + if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 { + t.Errorf("across a wrap: %.2f W, expected 50", got) + } +} + +func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) { + root := t.TempDir() + put(t, root, "class/thermal/cooling_device0/type", "Processor") + now := time.Now() + got := sampler(root, noSMI, &now).Sample(t.Context()) + if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil { + t.Errorf("a machine with no sensors reported %+v", got) + } +} + +func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) { + root := t.TempDir() + put(t, root, "class/hwmon/hwmon2/name", "nct6798") + put(t, root, "class/hwmon/hwmon2/temp1_input", "24500") + put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN") + put(t, root, "class/hwmon/hwmon3/name", "k10temp") + put(t, root, "class/hwmon/hwmon3/temp1_input", "71250") + put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl") + now := time.Now() + got := sampler(root, noSMI, &now).Sample(t.Context()) + if got.InletTempC != 24.5 || got.CPUTempC != 71.25 { + t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC) + } +} + +func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) { + var asked []string + smi := func(_ context.Context, fields string) ([]byte, error) { + asked = append(asked, fields) + return []byte( + "0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" + + "1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal + "2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only + } + now := time.Now() + ss := sampler(t.TempDir(), smi, &now) + got := ss.Sample(t.Context()) + if len(got.GPUs) != 3 { + t.Fatalf("%+v", got.GPUs) + } + g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2] + if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil { + t.Errorf("card 0 %+v", g0) + } + if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" { + t.Errorf("card 1 throttled by %v", g1.Throttle) + } + if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil { + t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2) + } + // Within ten seconds the last reading is reused rather than running + // nvidia-smi on every heartbeat; after, it is asked again. + now = now.Add(5 * time.Second) + ss.Sample(t.Context()) + if len(asked) != 1 { + t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked)) + } + now = now.Add(6 * time.Second) + ss.Sample(t.Context()) + if len(asked) != 2 { + t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked)) + } +} + +func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) { + smi := func(_ context.Context, fields string) ([]byte, error) { + if strings.Contains(fields, "temperature.memory") { + return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`) + } + return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil + } + now := time.Now() + got := sampler(t.TempDir(), smi, &now).Sample(t.Context()) + if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" { + t.Errorf("%+v", got.GPUs) + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index d2da729..ed7bd05 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -249,6 +249,11 @@ type LoadView struct { // MemPercent is used over used plus available. MemPercent float64 `json:"memory_percent"` AgeSec int64 `json:"age_seconds"` + // What the node's sensors say; zero is not measured. + CPUTempC float64 `json:"cpu_temp_c,omitempty"` + CPUPowerW float64 `json:"cpu_power_w,omitempty"` + InletTempC float64 `json:"inlet_temp_c,omitempty"` + GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"` } // loadFresh is how old a measurement may be and still be shown as current. @@ -262,7 +267,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView { CPUPercent: l.CPUPercent, CPUs: l.CPUCount, Load1: l.Load1, Load5: l.Load5, Load15: l.Load15, MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes, - AgeSec: int64(now.Sub(l.At).Seconds()), + AgeSec: int64(now.Sub(l.At).Seconds()), + CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs, } if v.MemTotal > 0 { v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal) diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 68de0e8..88c44b9 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -545,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(), } } + out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC() + for _, g := range l.GetGpus() { + out.GPUs = append(out.GPUs, inventory.GPUSensor{ + Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(), + PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(), + }) + } return out } diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 37bd848..81f5923 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -74,6 +74,13 @@ type Collector struct { gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc gpuTemp *prometheus.Desc + gpuMemTemp *prometheus.Desc + gpuPower *prometheus.Desc + gpuPowerLimit *prometheus.Desc + gpuThrottled *prometheus.Desc + nodeCPUTemp *prometheus.Desc + nodeCPUPower *prometheus.Desc + nodeInletTemp *prometheus.Desc gpuHealthy *prometheus.Desc gpuNVLink *prometheus.Desc gpuECC *prometheus.Desc @@ -142,6 +149,21 @@ func New(src Source) *Collector { "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), + gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius", + "A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil), + gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts", + "A GPU's power draw now.", gpu, nil), + gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts", + "The power limit a GPU is held to.", gpu, nil), + gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled", + "1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.", + append(gpu, "reason"), nil), + nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius", + "A node's hottest CPU sensor.", node, nil), + nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts", + "A node's CPU packages' power draw, from RAPL.", node, nil), + nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius", + "A node's inlet or ambient air sensor, where its board has one.", node, nil), gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius", "A GPU's reported temperature.", gpu, nil), gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy", @@ -250,6 +272,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, + c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted, @@ -297,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { } if l := n.Load; l != nil && now.Sub(l.At) < loadFresh { + // Each only when measured: a VM has none of these, and a zero + // would read as a machine at 0 °C drawing nothing. + for _, v := range []struct { + d *prometheus.Desc + val float64 + }{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} { + if v.val > 0 { + ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID) + } + } ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID) if l.CPUCount > 0 { ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID) @@ -366,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, float64(g.UtilizationPercent), labels...) } - if g.Health.TemperatureC > 0 { + // The sensors' fresh reading when there is one; the temperature + // read at registration otherwise, which is all an older agent sends. + if s, ok := sensorFor(n.Load, g.Index, now); ok { + c.gpuSensor(ch, s, labels) + } else if g.Health.TemperatureC > 0 { ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue, float64(g.Health.TemperatureC), labels...) } @@ -518,3 +555,30 @@ func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) { } } } + +// sensorFor is a GPU's fresh sensor reading, if the node sent one. +func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) { + if l == nil || now.Sub(l.At) >= loadFresh { + return inventory.GPUSensor{}, false + } + for _, g := range l.GPUs { + if g.Index == index { + return g, true + } + } + return inventory.GPUSensor{}, false +} + +func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) { + for _, v := range []struct { + d *prometheus.Desc + val float64 + }{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} { + if v.val > 0 { + ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...) + } + } + for _, reason := range s.Throttle { + ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...) + } +} diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go new file mode 100644 index 0000000..cd73f6b --- /dev/null +++ b/internal/control/fleetmetrics/sensors_test.go @@ -0,0 +1,41 @@ +package fleetmetrics + +import ( + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) { + gpuNode := inventory.Node{ID: "g1", State: inventory.Ready, + GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}}, + Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65, + GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700, + Throttle: []string{"sw_power_cap", "sw_thermal"}}}}} + vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}} + out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}}) + + for _, want := range []string{ + "llmgw_node_cpu_temperature_celsius node=g1 -> 58", + "llmgw_node_cpu_power_watts node=g1 -> 65", + "llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698", + "llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700", + "llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1", + // The fresh reading, not the 40 °C from registration. + "llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q", want) + } + } + for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} { + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) { + t.Errorf("published what was not measured: %s", line) + } + } + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index dc92159..4e9df64 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -304,6 +304,21 @@ type Load struct { MemAvailBytes int64 // Shards is each engine's use, by shard ID. Shards map[string]ShardLoad + // What the node's sensors say; zero is not measured. + CPUTempC float64 + CPUPowerW float64 + InletTempC float64 + GPUs []GPUSensor +} + +// GPUSensor is one card's heat and power, and what is throttling it. +type GPUSensor struct { + Index int `json:"index"` + TempC float64 `json:"temp_c,omitempty"` + MemoryTempC float64 `json:"memory_temp_c,omitempty"` + PowerW float64 `json:"power_w,omitempty"` + PowerLimitW float64 `json:"power_limit_w,omitempty"` + Throttle []string `json:"throttle,omitempty"` } // ShardLoad is one engine's use: cores busy, and resident memory. diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index e110b32..6e91dbd 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -7,6 +7,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/control/topology" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" @@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) { t.Error("the cluster page does not lead to the racks") } } + +// hotSource is a GPU node whose sensors report, one card held back. +type hotSource struct{ fakeSource } + +func (hotSource) Nodes() []adminapi.NodeView { + n := fakeSource{}.Nodes()[0] + n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39, + CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5, + GPUs: []inventory.GPUSensor{ + {Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700}, + {Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}}, + }} + return []adminapi.NodeView{n} +} + +func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) { + _, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil) + for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700", + `sw_power_cap, sw_thermal`} { + if !strings.Contains(body, want) { + t.Errorf("the page lacks %q", want) + } + } + _, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil) + for _, absent := range []string{"°C", "
{{.}}
{{end}}{{end}} {{if .Unhealthy}}{{join .Unhealthy}}
{{end}} +{{with .Load}}{{if .GPUs}} +| GPU | Temperature | Memory | Power | Held back by |
|---|---|---|---|---|
| {{.Index}} | +{{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}} | +{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}} | +{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}{{printf "%.0f" .PowerW}} W of {{printf "%.0f" .PowerLimitW}}{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}} | +{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}} | +