Read nodes' heat and power: GPUs, CPUs, board inlet, RAPL
Release / build (push) Successful in 1m10s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s

Every agent samples, at most every ten seconds: nvidia-smi's temperature,
HBM temperature, power draw, enforced limit and throttle reasons per GPU;
hwmon's CPU package and board inlet temperatures; and CPU package power
from RAPL's energy counters. Sent with heartbeats, published as metrics
and shown on the node page; unmeasured sensors are absent, never zero.
Observe-only, step 0 of docs/facility-aware-placement.md, which now
describes the agent source and what these readings can and cannot say.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 19:49:08 +02:00
1 parent 8724dc027a
commit cd10e086c8
15 files changed
+996 -172

No files matched your search

+21
View File
@@ -132,6 +132,27 @@ decisions rather than only being printed:
A model never measured behaves exactly as before. Measurements are read from
the store every half minute.
### Nodes report their own heat and power
The first step of facility-aware placement (docs/facility-aware-placement.md),
observe-only: nothing is decided on these yet. Every agent reads, at most every
ten seconds, and sends with its heartbeat:
- **GPUs** from nvidia-smi: temperature, HBM temperature where the driver has
it, power draw and the enforced limit, and what is holding the clocks down —
power cap, thermal slowdown, power brake. A GPU's temperature was read once,
at registration; it is now current.
- **CPUs** from hwmon, which is what lm_sensors reads, with nothing to install:
the package temperature, and the board's inlet air sensor where it has one.
- **CPU power** from RAPL's energy counters.
A sensor a machine does not have is not reported, never shown as zero. New
metrics: `llmgw_gpu_power_watts`, `llmgw_gpu_power_limit_watts`,
`llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_throttled{reason}`,
`llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts`,
`llmgw_node_inlet_temperature_celsius`. Each node's page shows them, with a GPU
table of temperature, power against its limit, and what is throttling it.
### The mirror page in tabs
/admin/mirror was one long page: what the mirror holds, the Hugging Face
+295 -157
View File
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19, 0}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -1415,8 +1415,15 @@ type NodeLoad struct {
MemoryUsedBytes int64 `protobuf:"varint,6,opt,name=memory_used_bytes,json=memoryUsedBytes,proto3" json:"memory_used_bytes,omitempty"`
MemoryAvailableBytes int64 `protobuf:"varint,7,opt,name=memory_available_bytes,json=memoryAvailableBytes,proto3" json:"memory_available_bytes,omitempty"`
Shards []*ShardLoad `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// What the machine's own sensors say; zero is not measured. cpu_temp_c is
// the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
// inlet_temp_c the board's inlet or ambient sensor where it has one.
CpuTempC float64 `protobuf:"fixed64,9,opt,name=cpu_temp_c,json=cpuTempC,proto3" json:"cpu_temp_c,omitempty"`
CpuPowerW float64 `protobuf:"fixed64,10,opt,name=cpu_power_w,json=cpuPowerW,proto3" json:"cpu_power_w,omitempty"`
InletTempC float64 `protobuf:"fixed64,11,opt,name=inlet_temp_c,json=inletTempC,proto3" json:"inlet_temp_c,omitempty"`
Gpus []*GPUSensor `protobuf:"bytes,12,rep,name=gpus,proto3" json:"gpus,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *NodeLoad) Reset() {
@@ -1505,6 +1512,121 @@ func (x *NodeLoad) GetShards() []*ShardLoad {
return nil
}
func (x *NodeLoad) GetCpuTempC() float64 {
if x != nil {
return x.CpuTempC
}
return 0
}
func (x *NodeLoad) GetCpuPowerW() float64 {
if x != nil {
return x.CpuPowerW
}
return 0
}
func (x *NodeLoad) GetInletTempC() float64 {
if x != nil {
return x.InletTempC
}
return 0
}
func (x *NodeLoad) GetGpus() []*GPUSensor {
if x != nil {
return x.Gpus
}
return nil
}
// GPUSensor is one card's heat, power, and what is holding it back.
type GPUSensor struct {
state protoimpl.MessageState `protogen:"open.v1"`
Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"`
TempC float64 `protobuf:"fixed64,2,opt,name=temp_c,json=tempC,proto3" json:"temp_c,omitempty"`
MemoryTempC float64 `protobuf:"fixed64,3,opt,name=memory_temp_c,json=memoryTempC,proto3" json:"memory_temp_c,omitempty"`
PowerW float64 `protobuf:"fixed64,4,opt,name=power_w,json=powerW,proto3" json:"power_w,omitempty"`
PowerLimitW float64 `protobuf:"fixed64,5,opt,name=power_limit_w,json=powerLimitW,proto3" json:"power_limit_w,omitempty"`
// throttle names what is holding the clocks down now: sw_power_cap,
// hw_slowdown, sw_thermal, hw_thermal, power_brake.
Throttle []string `protobuf:"bytes,6,rep,name=throttle,proto3" json:"throttle,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *GPUSensor) Reset() {
*x = GPUSensor{}
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *GPUSensor) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*GPUSensor) ProtoMessage() {}
func (x *GPUSensor) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use GPUSensor.ProtoReflect.Descriptor instead.
func (*GPUSensor) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
}
func (x *GPUSensor) GetIndex() int32 {
if x != nil {
return x.Index
}
return 0
}
func (x *GPUSensor) GetTempC() float64 {
if x != nil {
return x.TempC
}
return 0
}
func (x *GPUSensor) GetMemoryTempC() float64 {
if x != nil {
return x.MemoryTempC
}
return 0
}
func (x *GPUSensor) GetPowerW() float64 {
if x != nil {
return x.PowerW
}
return 0
}
func (x *GPUSensor) GetPowerLimitW() float64 {
if x != nil {
return x.PowerLimitW
}
return 0
}
func (x *GPUSensor) GetThrottle() []string {
if x != nil {
return x.Throttle
}
return nil
}
// ShardLoad is one engine process group's use.
type ShardLoad struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -1520,7 +1642,7 @@ type ShardLoad struct {
func (x *ShardLoad) Reset() {
*x = ShardLoad{}
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1532,7 +1654,7 @@ func (x *ShardLoad) String() string {
func (*ShardLoad) ProtoMessage() {}
func (x *ShardLoad) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1545,7 +1667,7 @@ func (x *ShardLoad) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardLoad.ProtoReflect.Descriptor instead.
func (*ShardLoad) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
}
func (x *ShardLoad) GetShardId() string {
@@ -1613,7 +1735,7 @@ type IBPort struct {
func (x *IBPort) Reset() {
*x = IBPort{}
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1625,7 +1747,7 @@ func (x *IBPort) String() string {
func (*IBPort) ProtoMessage() {}
func (x *IBPort) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1638,7 +1760,7 @@ func (x *IBPort) ProtoReflect() protoreflect.Message {
// Deprecated: Use IBPort.ProtoReflect.Descriptor instead.
func (*IBPort) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
}
func (x *IBPort) GetHca() string {
@@ -1781,7 +1903,7 @@ type SubnetManager struct {
func (x *SubnetManager) Reset() {
*x = SubnetManager{}
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1793,7 +1915,7 @@ func (x *SubnetManager) String() string {
func (*SubnetManager) ProtoMessage() {}
func (x *SubnetManager) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1806,7 +1928,7 @@ func (x *SubnetManager) ProtoReflect() protoreflect.Message {
// Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead.
func (*SubnetManager) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *SubnetManager) GetGuid() string {
@@ -1849,7 +1971,7 @@ type IMEX struct {
func (x *IMEX) Reset() {
*x = IMEX{}
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1861,7 +1983,7 @@ func (x *IMEX) String() string {
func (*IMEX) ProtoMessage() {}
func (x *IMEX) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1874,7 +1996,7 @@ func (x *IMEX) ProtoReflect() protoreflect.Message {
// Deprecated: Use IMEX.ProtoReflect.Descriptor instead.
func (*IMEX) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
}
func (x *IMEX) GetState() string {
@@ -1906,7 +2028,7 @@ type GPUFabricState struct {
func (x *GPUFabricState) Reset() {
*x = GPUFabricState{}
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1918,7 +2040,7 @@ func (x *GPUFabricState) String() string {
func (*GPUFabricState) ProtoMessage() {}
func (x *GPUFabricState) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1931,7 +2053,7 @@ func (x *GPUFabricState) ProtoReflect() protoreflect.Message {
// Deprecated: Use GPUFabricState.ProtoReflect.Descriptor instead.
func (*GPUFabricState) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
}
func (x *GPUFabricState) GetIndex() int32 {
@@ -2038,7 +2160,7 @@ type Heartbeat struct {
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2050,7 +2172,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2063,7 +2185,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
}
func (x *Heartbeat) GetNodeId() string {
@@ -2211,7 +2333,7 @@ type Provision struct {
func (x *Provision) Reset() {
*x = Provision{}
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2223,7 +2345,7 @@ func (x *Provision) String() string {
func (*Provision) ProtoMessage() {}
func (x *Provision) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2236,7 +2358,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message {
// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
func (*Provision) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
}
func (x *Provision) GetEngines() []*EngineWant {
@@ -2257,7 +2379,7 @@ type EngineWant struct {
func (x *EngineWant) Reset() {
*x = EngineWant{}
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2269,7 +2391,7 @@ func (x *EngineWant) String() string {
func (*EngineWant) ProtoMessage() {}
func (x *EngineWant) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2282,7 +2404,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
func (*EngineWant) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
}
func (x *EngineWant) GetEngine() string {
@@ -2315,7 +2437,7 @@ type ProvisionStatus struct {
func (x *ProvisionStatus) Reset() {
*x = ProvisionStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2327,7 +2449,7 @@ func (x *ProvisionStatus) String() string {
func (*ProvisionStatus) ProtoMessage() {}
func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2340,7 +2462,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
func (*ProvisionStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
}
func (x *ProvisionStatus) GetEngine() string {
@@ -2404,7 +2526,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2416,7 +2538,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2429,7 +2551,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
}
func (x *ShardStatus) GetNodeId() string {
@@ -2503,7 +2625,7 @@ type FetchProgress struct {
func (x *FetchProgress) Reset() {
*x = FetchProgress{}
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2515,7 +2637,7 @@ func (x *FetchProgress) String() string {
func (*FetchProgress) ProtoMessage() {}
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2528,7 +2650,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
func (*FetchProgress) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
}
func (x *FetchProgress) GetSource() string {
@@ -2591,7 +2713,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2603,7 +2725,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2616,7 +2738,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
}
func (x *StartShard) GetReplicaId() string {
@@ -2757,7 +2879,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2769,7 +2891,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2782,7 +2904,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
}
func (x *EngineProfile) GetName() string {
@@ -2878,7 +3000,7 @@ type RemoveWeights struct {
func (x *RemoveWeights) Reset() {
*x = RemoveWeights{}
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2890,7 +3012,7 @@ func (x *RemoveWeights) String() string {
func (*RemoveWeights) ProtoMessage() {}
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2903,7 +3025,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
func (*RemoveWeights) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
}
func (x *RemoveWeights) GetPaths() []string {
@@ -2923,7 +3045,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2935,7 +3057,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2948,7 +3070,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
}
func (x *StopShard) GetShardId() string {
@@ -2968,7 +3090,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2980,7 +3102,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2993,7 +3115,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26}
}
func (x *DrainShard) GetShardId() string {
@@ -3015,7 +3137,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3027,7 +3149,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3040,7 +3162,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27}
}
func (x *Ack) GetMessageId() string {
@@ -3084,7 +3206,7 @@ type LogsRequest struct {
func (x *LogsRequest) Reset() {
*x = LogsRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3096,7 +3218,7 @@ func (x *LogsRequest) String() string {
func (*LogsRequest) ProtoMessage() {}
func (x *LogsRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[27]
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3109,7 +3231,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead.
func (*LogsRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{27}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28}
}
func (x *LogsRequest) GetRequestId() string {
@@ -3149,7 +3271,7 @@ type LogsReply struct {
func (x *LogsReply) Reset() {
*x = LogsReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
mi := &file_llmgw_v1_agent_proto_msgTypes[29]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3161,7 +3283,7 @@ func (x *LogsReply) String() string {
func (*LogsReply) ProtoMessage() {}
func (x *LogsReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[28]
mi := &file_llmgw_v1_agent_proto_msgTypes[29]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3174,7 +3296,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use LogsReply.ProtoReflect.Descriptor instead.
func (*LogsReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{28}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29}
}
func (x *LogsReply) GetRequestId() string {
@@ -3220,7 +3342,7 @@ type FabricScanRequest struct {
func (x *FabricScanRequest) Reset() {
*x = FabricScanRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[29]
mi := &file_llmgw_v1_agent_proto_msgTypes[30]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3232,7 +3354,7 @@ func (x *FabricScanRequest) String() string {
func (*FabricScanRequest) ProtoMessage() {}
func (x *FabricScanRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[29]
mi := &file_llmgw_v1_agent_proto_msgTypes[30]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3245,7 +3367,7 @@ func (x *FabricScanRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricScanRequest.ProtoReflect.Descriptor instead.
func (*FabricScanRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{29}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30}
}
func (x *FabricScanRequest) GetRequestId() string {
@@ -3284,7 +3406,7 @@ type FabricScanReply struct {
func (x *FabricScanReply) Reset() {
*x = FabricScanReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[30]
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3296,7 +3418,7 @@ func (x *FabricScanReply) String() string {
func (*FabricScanReply) ProtoMessage() {}
func (x *FabricScanReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[30]
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3309,7 +3431,7 @@ func (x *FabricScanReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricScanReply.ProtoReflect.Descriptor instead.
func (*FabricScanReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{30}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31}
}
func (x *FabricScanReply) GetRequestId() string {
@@ -3365,7 +3487,7 @@ type FabricTestRequest struct {
func (x *FabricTestRequest) Reset() {
*x = FabricTestRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3377,7 +3499,7 @@ func (x *FabricTestRequest) String() string {
func (*FabricTestRequest) ProtoMessage() {}
func (x *FabricTestRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3390,7 +3512,7 @@ func (x *FabricTestRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead.
func (*FabricTestRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32}
}
func (x *FabricTestRequest) GetRequestId() string {
@@ -3459,7 +3581,7 @@ type FabricTestReply struct {
func (x *FabricTestReply) Reset() {
*x = FabricTestReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3471,7 +3593,7 @@ func (x *FabricTestReply) String() string {
func (*FabricTestReply) ProtoMessage() {}
func (x *FabricTestReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3484,7 +3606,7 @@ func (x *FabricTestReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead.
func (*FabricTestReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33}
}
func (x *FabricTestReply) GetRequestId() string {
@@ -3544,7 +3666,7 @@ type UpgradeAgent struct {
func (x *UpgradeAgent) Reset() {
*x = UpgradeAgent{}
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3556,7 +3678,7 @@ func (x *UpgradeAgent) String() string {
func (*UpgradeAgent) ProtoMessage() {}
func (x *UpgradeAgent) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[33]
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3569,7 +3691,7 @@ func (x *UpgradeAgent) ProtoReflect() protoreflect.Message {
// Deprecated: Use UpgradeAgent.ProtoReflect.Descriptor instead.
func (*UpgradeAgent) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{33}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34}
}
func (x *UpgradeAgent) GetRequestId() string {
@@ -3606,7 +3728,7 @@ type UpgradeReply struct {
func (x *UpgradeReply) Reset() {
*x = UpgradeReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
mi := &file_llmgw_v1_agent_proto_msgTypes[35]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -3618,7 +3740,7 @@ func (x *UpgradeReply) String() string {
func (*UpgradeReply) ProtoMessage() {}
func (x *UpgradeReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[34]
mi := &file_llmgw_v1_agent_proto_msgTypes[35]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3631,7 +3753,7 @@ func (x *UpgradeReply) ProtoReflect() protoreflect.Message {
// Deprecated: Use UpgradeReply.ProtoReflect.Descriptor instead.
func (*UpgradeReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{34}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{35}
}
func (x *UpgradeReply) GetRequestId() string {
@@ -3786,7 +3908,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"\x9b\x02\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xa4\x03\n" +
"\bNodeLoad\x12\x1f\n" +
"\vcpu_percent\x18\x01 \x01(\x01R\n" +
"cpuPercent\x12\x1b\n" +
@@ -3796,7 +3918,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x06load15\x18\x05 \x01(\x01R\x06load15\x12*\n" +
"\x11memory_used_bytes\x18\x06 \x01(\x03R\x0fmemoryUsedBytes\x124\n" +
"\x16memory_available_bytes\x18\a \x01(\x03R\x14memoryAvailableBytes\x12+\n" +
"\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\"m\n" +
"\x06shards\x18\b \x03(\v2\x13.llmgw.v1.ShardLoadR\x06shards\x12\x1c\n" +
"\n" +
"cpu_temp_c\x18\t \x01(\x01R\bcpuTempC\x12\x1e\n" +
"\vcpu_power_w\x18\n" +
" \x01(\x01R\tcpuPowerW\x12 \n" +
"\finlet_temp_c\x18\v \x01(\x01R\n" +
"inletTempC\x12'\n" +
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xb5\x01\n" +
"\tGPUSensor\x12\x14\n" +
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" +
"\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" +
"\rmemory_temp_c\x18\x03 \x01(\x01R\vmemoryTempC\x12\x17\n" +
"\apower_w\x18\x04 \x01(\x01R\x06powerW\x12\"\n" +
"\rpower_limit_w\x18\x05 \x01(\x01R\vpowerLimitW\x12\x1a\n" +
"\bthrottle\x18\x06 \x03(\tR\bthrottle\"m\n" +
"\tShardLoad\x12\x19\n" +
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
@@ -4026,7 +4162,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 41)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -4039,91 +4175,93 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment
(*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
(*IBPort)(nil), // 12: llmgw.v1.IBPort
(*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager
(*IMEX)(nil), // 14: llmgw.v1.IMEX
(*GPUFabricState)(nil), // 15: llmgw.v1.GPUFabricState
(*Heartbeat)(nil), // 16: llmgw.v1.Heartbeat
(*Provision)(nil), // 17: llmgw.v1.Provision
(*EngineWant)(nil), // 18: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 19: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 20: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 21: llmgw.v1.FetchProgress
(*StartShard)(nil), // 22: llmgw.v1.StartShard
(*EngineProfile)(nil), // 23: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 24: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 25: llmgw.v1.StopShard
(*DrainShard)(nil), // 26: llmgw.v1.DrainShard
(*Ack)(nil), // 27: llmgw.v1.Ack
(*LogsRequest)(nil), // 28: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 29: llmgw.v1.LogsReply
(*FabricScanRequest)(nil), // 30: llmgw.v1.FabricScanRequest
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
(*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest
(*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply
(*UpgradeAgent)(nil), // 34: llmgw.v1.UpgradeAgent
(*UpgradeReply)(nil), // 35: llmgw.v1.UpgradeReply
nil, // 36: llmgw.v1.Register.LabelsEntry
nil, // 37: llmgw.v1.Register.EngineVersionsEntry
nil, // 38: llmgw.v1.GPU.NicPathsEntry
nil, // 39: llmgw.v1.IBPort.CountersEntry
nil, // 40: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 41: llmgw.v1.EngineProfile.EnvEntry
(*GPUSensor)(nil), // 11: llmgw.v1.GPUSensor
(*ShardLoad)(nil), // 12: llmgw.v1.ShardLoad
(*IBPort)(nil), // 13: llmgw.v1.IBPort
(*SubnetManager)(nil), // 14: llmgw.v1.SubnetManager
(*IMEX)(nil), // 15: llmgw.v1.IMEX
(*GPUFabricState)(nil), // 16: llmgw.v1.GPUFabricState
(*Heartbeat)(nil), // 17: llmgw.v1.Heartbeat
(*Provision)(nil), // 18: llmgw.v1.Provision
(*EngineWant)(nil), // 19: llmgw.v1.EngineWant
(*ProvisionStatus)(nil), // 20: llmgw.v1.ProvisionStatus
(*ShardStatus)(nil), // 21: llmgw.v1.ShardStatus
(*FetchProgress)(nil), // 22: llmgw.v1.FetchProgress
(*StartShard)(nil), // 23: llmgw.v1.StartShard
(*EngineProfile)(nil), // 24: llmgw.v1.EngineProfile
(*RemoveWeights)(nil), // 25: llmgw.v1.RemoveWeights
(*StopShard)(nil), // 26: llmgw.v1.StopShard
(*DrainShard)(nil), // 27: llmgw.v1.DrainShard
(*Ack)(nil), // 28: llmgw.v1.Ack
(*LogsRequest)(nil), // 29: llmgw.v1.LogsRequest
(*LogsReply)(nil), // 30: llmgw.v1.LogsReply
(*FabricScanRequest)(nil), // 31: llmgw.v1.FabricScanRequest
(*FabricScanReply)(nil), // 32: llmgw.v1.FabricScanReply
(*FabricTestRequest)(nil), // 33: llmgw.v1.FabricTestRequest
(*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply
(*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent
(*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply
nil, // 37: llmgw.v1.Register.LabelsEntry
nil, // 38: llmgw.v1.Register.EngineVersionsEntry
nil, // 39: llmgw.v1.GPU.NicPathsEntry
nil, // 40: llmgw.v1.IBPort.CountersEntry
nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 42: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
16, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
20, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
35, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply
22, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
25, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
26, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
27, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
24, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
17, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
28, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
30, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
32, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
34, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
36, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
17, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
21, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
30, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
32, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
34, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
36, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply
23, // 7: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
26, // 8: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
27, // 9: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
28, // 10: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
25, // 11: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
18, // 12: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
29, // 13: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
37, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
14, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
38, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
39, // 29: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 30: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 31: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 32: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
40, // 33: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
19, // 34: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 35: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 36: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 37: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 38: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
14, // 39: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
18, // 40: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 41: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
21, // 42: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
23, // 43: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
41, // 44: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 45: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 46: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
46, // [46:47] is the sub-list for method output_type
45, // [45:46] is the sub-list for method input_type
45, // [45:45] is the sub-list for extension type_name
45, // [45:45] is the sub-list for extension extendee
0, // [0:45] is the sub-list for field type_name
39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 38: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
16, // 39: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
15, // 40: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
19, // 41: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
47, // [47:48] is the sub-list for method output_type
46, // [46:47] is the sub-list for method input_type
46, // [46:46] is the sub-list for extension type_name
46, // [46:46] is the sub-list for extension extendee
0, // [0:46] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -4158,7 +4296,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 41,
NumMessages: 42,
NumExtensions: 0,
NumServices: 1,
},
+19
View File
@@ -285,6 +285,25 @@ message NodeLoad {
int64 memory_used_bytes = 6;
int64 memory_available_bytes = 7;
repeated ShardLoad shards = 8;
// What the machine's own sensors say; zero is not measured. cpu_temp_c is
// the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
// inlet_temp_c the board's inlet or ambient sensor where it has one.
double cpu_temp_c = 9;
double cpu_power_w = 10;
double inlet_temp_c = 11;
repeated GPUSensor gpus = 12;
}
// GPUSensor is one card's heat, power, and what is holding it back.
message GPUSensor {
int32 index = 1;
double temp_c = 2;
double memory_temp_c = 3;
double power_w = 4;
double power_limit_w = 5;
// throttle names what is holding the clocks down now: sw_power_cap,
// hw_slowdown, sw_thermal, hw_thermal, power_brake.
repeated string throttle = 6;
}
// ShardLoad is one engine process group's use.
+48 -13
View File
@@ -1,9 +1,11 @@
# Facility-aware placement: rack power, temperature and cooling
**Status: planned, not built.** This is the design for step 5 of resource
allocation, after reserved CPU cores and memory (1), capacity classes and
preemption (2), tenant capacity quotas (3) and sizing by measured throughput
(4), all of which shipped in v0.10.5. Nothing here exists in the code yet.
**Status: planned; its first step, reading the nodes' own sensors, is built.**
This is the design for step 5 of resource allocation, after reserved CPU cores
and memory (1), capacity classes and preemption (2), tenant capacity quotas (3)
and sizing by measured throughput (4), all of which shipped in v0.10.5. Of
what follows, only the node sensors (rollout step 0) exist in the code: they
are read, published and shown, and change no decision.
## Why
@@ -35,7 +37,7 @@ and never let missing data stop the cluster.
| Headroom: the scheduler dry-run on a copy of the inventory | Room that respects power budgets, for free |
| Preemption with classes, minimums and a dry run (step 2) | Shedding load from a hot rack in the right order |
| Measured throughput per profile (step 4) | Tokens per watt as a placement preference |
| GPU temperature from NVML, per card | A node-level thermal signal that needs no facility data |
| Node sensors, read by every agent (step 0, built) | Heat and power with no facility integration: see below |
| `internal/promq`, a Prometheus client | Reading facility metrics that DCIM exporters publish |
## Concepts
@@ -116,8 +118,8 @@ starts nor ends anything:
| Condition | Enters when | Placement |
|---|---|---|
| `ok` | below the warm threshold for 5 min | normal |
| `warm` | inlet above the warm threshold for 2 min, or GPU temperatures near throttling | allowed, scored down: other domains first |
| `hot` | inlet above the hot threshold for 2 min, or a cooling unit in alarm | no new placements, as a cordon would, but automatic and reversible |
| `warm` | inlet above the warm threshold for 2 min, GPU temperatures near throttling, or any GPU thermally throttled | allowed, scored down: other domains first |
| `hot` | inlet above the hot threshold for 2 min, a cooling unit in alarm, or a sustained share of the domain's GPUs thermally throttled | no new placements, as a cordon would, but automatic and reversible |
| `unknown` | no fresh reading | as `ok`, and reported as unknown |
**Shedding** (optional, off by default): a domain `hot` for longer than its
@@ -186,9 +188,39 @@ type FacilitySource interface {
query: max by (row) (crac_alarm)
domain_label: row
```
- **Agents.** What the nodes can read themselves: GPU power and temperature from
NVML, node power from Redfish or IPMI where the BMC allows it. Needs no
facility integration, and gives node-level signals on day one.
- **Agents (built, step 0).** What each node reads about itself, every ten
seconds at most, sent with its heartbeat:
- **GPUs, from nvidia-smi:** temperature, memory (HBM) temperature where the
driver reports it, power draw, the enforced power limit, and what is
holding the clocks down now — `sw_power_cap`, `hw_slowdown`, `sw_thermal`,
`hw_thermal`, `power_brake`.
- **CPUs, from hwmon** (what lm_sensors reads, without it installed): the
package temperature, or the hottest core; and the board's inlet or ambient
sensor where it has one (`SYSTIN`, `Inlet`, `Ambient`).
- **CPU power, from RAPL:** the packages' energy counters over time,
subdomains not counted twice, counter wrap handled.
A machine without a sensor reports nothing for it, never a zero; a VM has
none of them. Published as `llmgw_gpu_temperature_celsius`,
`llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_power_watts`,
`llmgw_gpu_power_limit_watts`, `llmgw_gpu_throttled{reason}`,
`llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts` and
`llmgw_node_inlet_temperature_celsius`, and on each node's page.
What they can and cannot say:
- **Throttling is the strongest signal there is.** A temperature is a proxy;
a card holding its clocks down is the fact that invalidates its measured
capacity, and it needs no threshold to be chosen.
- **Measured power** is GPU draw plus CPU packages; memory, disks, fans and
power-supply losses are not in it, so a node's draw is that plus an
estimated remainder. A rack's is its llmgw nodes' plus a declared figure
for everything else in it: switches and storage report nothing here.
- **CPU and GPU temperatures follow the load, not the room.** A hot rack is
inferred from many nodes running hot at comparable load, or from
throttling, not from one node's CPU. Only a board inlet sensor reads the
air, and most boards have none.
- **No budget comes from a sensor.** A reading is a draw; the cap it is
measured against still comes from the facility.
- **Simulated.** Readings from a script, for tests and for rehearsing a
thermal event on a cluster without facility data.
@@ -223,14 +255,17 @@ type FacilitySource interface {
Each step ships on its own and is useful without the next:
0. **Node sensors, observe-only (built).** GPU heat, power and throttling,
CPU heat and power, inlet air where the board has a sensor; published and
shown, no decision changed.
1. **Location and spreading.** Well-known labels, domains from labels, replicas
spread across racks and PDUs, locations on the node views. No facility data
needed.
2. **Power budgets from a static file.** Caps and margins, estimated draw per
replica (declared or derived), hard limit in placement and headroom.
3. **Readings, observe-only.** The Prometheus and agent sources, conditions
with hysteresis, measured power folded into headroom; decisions shown, not
enforced.
3. **Readings, observe-only.** The Prometheus source beside the agents',
conditions with hysteresis (throttling first among them), measured power
folded into headroom; decisions shown, not enforced.
4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and
release.
5. **Shedding and tokens per watt.** Opt-in shedding through step 2's
+14
View File
@@ -101,6 +101,9 @@ type Agent struct {
smsErr map[string]string
// load measures the machine between heartbeats.
load *discovery.LoadSampler
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
// from nvidia-smi.
sensors *discovery.SensorSampler
// fetchWatchers are the shards waiting on each source, so progress on one
// download reaches every shard that asked for it. A fetch is single-flight
@@ -146,6 +149,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
logRing: ring,
provKick: make(chan struct{}, 1),
load: discovery.NewLoadSampler(),
sensors: discovery.NewSensorSampler(),
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -638,6 +642,16 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
})
}
if a.sensors != nil {
se := a.sensors.Sample(context.Background())
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
for _, g := range se.GPUs {
out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
})
}
}
return out
}
+242
View File
@@ -0,0 +1,242 @@
package discovery
import (
"context"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// Sensors is what a node's own hardware says about heat and power: the CPUs,
// the board where it has an inlet sensor, and each GPU. A zero is not
// measured, never a reading of zero — a VM has none of these, and must not
// look like a machine running at 0 °C on no power.
type Sensors struct {
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
// where no package reading exists.
CPUTempC float64
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
// energy counters over the time between two samples.
CPUPowerW float64
// InletTempC is the air coming in, where the board has a sensor it calls
// that. Most do not, and a CPU's temperature is not a substitute: it
// follows the load, not the room.
InletTempC float64
GPUs []GPUSensor
}
// GPUSensor is one card's heat, power and whether it is slowing itself down.
type GPUSensor struct {
Index int
TempC float64
MemoryTempC float64
PowerW float64
PowerLimitW float64
// Throttle names what is holding the card's clocks down now:
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
// is running free. Idle and application clock settings are left out;
// they are not the card being held back.
Throttle []string
}
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
// than temperatures change, and each run costs a few hundred milliseconds.
const sensorEvery = 10 * time.Second
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
// two readings of the energy counters.
type SensorSampler struct {
sys string // /sys, or a test's copy of it
smi func(ctx context.Context, fields string) ([]byte, error)
now func() time.Time
mu sync.Mutex
last Sensors
lastAt time.Time
prevEnergy map[string]uint64
prevAt time.Time
}
// NewSensorSampler reads this machine.
func NewSensorSampler() *SensorSampler {
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
}
func runSMI(ctx context.Context, fields string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
//nolint:gosec // fixed binary, fields from this file
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
"--format=csv,noheader,nounits").Output()
}
// Sample reads the sensors, or returns the last reading when that is recent.
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
s.mu.Lock()
defer s.mu.Unlock()
now := s.now()
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
return s.last
}
out := Sensors{}
out.CPUTempC, out.InletTempC = s.hwmon()
out.CPUPowerW = s.rapl(now)
out.GPUs = s.gpus(ctx)
s.last, s.lastAt = out, now
return out
}
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
// device the kernel exposes — what lm_sensors reads, without it installed.
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
var pkg, core float64
for _, d := range devs {
name := readTrim(filepath.Join(d, "name"))
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
for _, in := range inputs {
milli, err := strconv.ParseFloat(readTrim(in), 64)
if err != nil || milli <= 0 {
continue
}
c := milli / 1000
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
switch {
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
pkg = max(pkg, c)
case isCPUSensor(name):
core = max(core, c)
case label == "cpu":
// A board's own reading of the CPU, as Dell's SMM gives.
core = max(core, c)
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
strings.Contains(label, "intake") || label == "systin":
inlet = max(inlet, c)
}
}
}
if pkg > 0 {
return pkg, inlet
}
return core, inlet
}
func isCPUSensor(name string) bool {
switch name {
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
return true
}
return false
}
// rapl is the CPU packages' power from their energy counters. Package
// domains only: the dram and core subdomains are parts of a package, and
// adding them would count the same energy twice.
func (s *SensorSampler) rapl(now time.Time) float64 {
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
energy := map[string]uint64{}
wrap := map[string]uint64{}
for _, d := range doms {
if strings.Count(filepath.Base(d), ":") != 1 {
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
}
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
if err != nil {
continue // unreadable without root on newer kernels
}
energy[d] = e
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
}
prev, prevAt := s.prevEnergy, s.prevAt
s.prevEnergy, s.prevAt = energy, now
dt := now.Sub(prevAt).Seconds()
if prev == nil || dt <= 0 || len(energy) == 0 {
return 0
}
var joules float64
for d, e := range energy {
p, ok := prev[d]
if !ok {
return 0 // a domain appeared: no fair difference this time
}
if e < p { // the counter wrapped
e += wrap[d]
}
joules += float64(e-p) / 1e6
}
return joules / dt
}
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
// card is being held back are named.
var throttleBits = []struct {
bit uint64
name string
}{
{0x4, "sw_power_cap"},
{0x8, "hw_slowdown"},
{0x20, "sw_thermal"},
{0x40, "hw_thermal"},
{0x80, "power_brake"},
}
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
// is not known to every driver, and a field nvidia-smi does not know fails the
// whole query, so it is asked for first and dropped on failure.
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
out, err := s.smi(ctx, withMem)
if err != nil {
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
return nil
}
}
var gs []GPUSensor
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
f := strings.Split(line, ",")
if len(f) < 5 {
continue
}
for i := range f {
f[i] = strings.TrimSpace(f[i])
}
idx, err := strconv.Atoi(f[0])
if err != nil {
continue
}
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
for _, b := range throttleBits {
if bits&b.bit != 0 {
g.Throttle = append(g.Throttle, b.name)
}
}
}
if len(f) > 5 {
g.MemoryTempC = num(f[5])
}
gs = append(gs, g)
}
return gs
}
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
func num(s string) float64 {
v, err := strconv.ParseFloat(s, 64)
if err != nil || v < 0 {
return 0
}
return v
}
func readTrim(path string) string {
b, err := os.ReadFile(path) //nolint:gosec // sysfs
if err != nil {
return ""
}
return strings.TrimSpace(string(b))
}
+169
View File
@@ -0,0 +1,169 @@
package discovery
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func put(t *testing.T, root, rel, body string) {
t.Helper()
p := filepath.Join(root, rel)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
t.Fatal(err)
}
}
// dellHost is the layout of the Proxmox host these tests were written
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
// and cores, and RAPL with a package and its dram subdomain.
func dellHost(t *testing.T) string {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
return root
}
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
}
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
s := sampler(root, noSMI, &now)
first := s.Sample(t.Context())
// The package reading, not the hotter core: the package is what the
// chip itself reports as its temperature.
if first.CPUTempC != 58 {
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
}
if first.CPUPowerW != 0 {
t.Error("power from one reading of a counter is invented")
}
if first.InletTempC != 0 || first.GPUs != nil {
t.Errorf("sensors this host does not have were reported: %+v", first)
}
// 10 s later the package has used 650 J, and its dram subdomain some of
// that — counted once.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
t.Errorf("CPU power %.2f W, expected 65", got)
}
}
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
root := dellHost(t)
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
s := sampler(root, noSMI, &now)
s.Sample(t.Context())
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
now = now.Add(10 * time.Second)
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
t.Errorf("across a wrap: %.2f W, expected 50", got)
}
}
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
root := t.TempDir()
put(t, root, "class/thermal/cooling_device0/type", "Processor")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
t.Errorf("a machine with no sensors reported %+v", got)
}
}
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
root := t.TempDir()
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
now := time.Now()
got := sampler(root, noSMI, &now).Sample(t.Context())
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
}
}
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
var asked []string
smi := func(_ context.Context, fields string) ([]byte, error) {
asked = append(asked, fields)
return []byte(
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
}
now := time.Now()
ss := sampler(t.TempDir(), smi, &now)
got := ss.Sample(t.Context())
if len(got.GPUs) != 3 {
t.Fatalf("%+v", got.GPUs)
}
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
t.Errorf("card 0 %+v", g0)
}
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
t.Errorf("card 1 throttled by %v", g1.Throttle)
}
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
}
// Within ten seconds the last reading is reused rather than running
// nvidia-smi on every heartbeat; after, it is asked again.
now = now.Add(5 * time.Second)
ss.Sample(t.Context())
if len(asked) != 1 {
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
}
now = now.Add(6 * time.Second)
ss.Sample(t.Context())
if len(asked) != 2 {
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
}
}
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
smi := func(_ context.Context, fields string) ([]byte, error) {
if strings.Contains(fields, "temperature.memory") {
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
}
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
}
now := time.Now()
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
t.Errorf("%+v", got.GPUs)
}
}
+7 -1
View File
@@ -249,6 +249,11 @@ type LoadView struct {
// MemPercent is used over used plus available.
MemPercent float64 `json:"memory_percent"`
AgeSec int64 `json:"age_seconds"`
// What the node's sensors say; zero is not measured.
CPUTempC float64 `json:"cpu_temp_c,omitempty"`
CPUPowerW float64 `json:"cpu_power_w,omitempty"`
InletTempC float64 `json:"inlet_temp_c,omitempty"`
GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"`
}
// loadFresh is how old a measurement may be and still be shown as current.
@@ -262,7 +267,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
AgeSec: int64(now.Sub(l.At).Seconds()),
AgeSec: int64(now.Sub(l.At).Seconds()),
CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs,
}
if v.MemTotal > 0 {
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
+7
View File
@@ -545,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
}
}
out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC()
for _, g := range l.GetGpus() {
out.GPUs = append(out.GPUs, inventory.GPUSensor{
Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(),
PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(),
})
}
return out
}
+65 -1
View File
@@ -74,6 +74,13 @@ type Collector struct {
gpuMemoryUsed *prometheus.Desc
gpuUtil *prometheus.Desc
gpuTemp *prometheus.Desc
gpuMemTemp *prometheus.Desc
gpuPower *prometheus.Desc
gpuPowerLimit *prometheus.Desc
gpuThrottled *prometheus.Desc
nodeCPUTemp *prometheus.Desc
nodeCPUPower *prometheus.Desc
nodeInletTemp *prometheus.Desc
gpuHealthy *prometheus.Desc
gpuNVLink *prometheus.Desc
gpuECC *prometheus.Desc
@@ -142,6 +149,21 @@ func New(src Source) *Collector {
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius",
"A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil),
gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts",
"A GPU's power draw now.", gpu, nil),
gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts",
"The power limit a GPU is held to.", gpu, nil),
gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled",
"1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.",
append(gpu, "reason"), nil),
nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius",
"A node's hottest CPU sensor.", node, nil),
nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts",
"A node's CPU packages' power draw, from RAPL.", node, nil),
nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius",
"A node's inlet or ambient air sensor, where its board has one.", node, nil),
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
"A GPU's reported temperature.", gpu, nil),
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
@@ -250,6 +272,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuNVLJoined, c.imexUp,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted,
@@ -297,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
}
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
// Each only when measured: a VM has none of these, and a zero
// would read as a machine at 0 °C drawing nothing.
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} {
if v.val > 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID)
}
}
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
if l.CPUCount > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
@@ -366,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
float64(g.UtilizationPercent), labels...)
}
if g.Health.TemperatureC > 0 {
// The sensors' fresh reading when there is one; the temperature
// read at registration otherwise, which is all an older agent sends.
if s, ok := sensorFor(n.Load, g.Index, now); ok {
c.gpuSensor(ch, s, labels)
} else if g.Health.TemperatureC > 0 {
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
float64(g.Health.TemperatureC), labels...)
}
@@ -518,3 +555,30 @@ func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) {
}
}
}
// sensorFor is a GPU's fresh sensor reading, if the node sent one.
func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) {
if l == nil || now.Sub(l.At) >= loadFresh {
return inventory.GPUSensor{}, false
}
for _, g := range l.GPUs {
if g.Index == index {
return g, true
}
}
return inventory.GPUSensor{}, false
}
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} {
if v.val > 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
}
}
for _, reason := range s.Throttle {
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
}
}
@@ -0,0 +1,41 @@
package fleetmetrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
for _, want := range []string{
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
"llmgw_node_cpu_power_watts node=g1 -> 65",
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
// The fresh reading, not the 40 °C from registration.
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q", want)
}
}
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
t.Errorf("published what was not measured: %s", line)
}
}
}
}
+15
View File
@@ -304,6 +304,21 @@ type Load struct {
MemAvailBytes int64
// Shards is each engine's use, by shard ID.
Shards map[string]ShardLoad
// What the node's sensors say; zero is not measured.
CPUTempC float64
CPUPowerW float64
InletTempC float64
GPUs []GPUSensor
}
// GPUSensor is one card's heat and power, and what is throttling it.
type GPUSensor struct {
Index int `json:"index"`
TempC float64 `json:"temp_c,omitempty"`
MemoryTempC float64 `json:"memory_temp_c,omitempty"`
PowerW float64 `json:"power_w,omitempty"`
PowerLimitW float64 `json:"power_limit_w,omitempty"`
Throttle []string `json:"throttle,omitempty"`
}
// ShardLoad is one engine's use: cores busy, and resident memory.
+31
View File
@@ -7,6 +7,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
@@ -174,3 +175,33 @@ func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
t.Error("the cluster page does not lead to the racks")
}
}
// hotSource is a GPU node whose sensors report, one card held back.
type hotSource struct{ fakeSource }
func (hotSource) Nodes() []adminapi.NodeView {
n := fakeSource{}.Nodes()[0]
n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
GPUs: []inventory.GPUSensor{
{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
}}
return []adminapi.NodeView{n}
}
func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
`<span class="bad">sw_power_cap, sw_thermal</span>`} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
if strings.Contains(vm, absent) {
t.Errorf("a node with no sensors shows %q", absent)
}
}
}
+15
View File
@@ -21,6 +21,10 @@
<div class="tile"><div class="v">{{printf "%.0f" .CPUPercent}}%</div><div class="k">CPU{{if .CPUs}} · {{.CPUs}} cores{{end}}</div>{{template "minibar" pctbar .CPUPercent}}</div>
<div class="tile"><div class="v">{{bytes .MemUsed}}</div><div class="k">RAM in use · of {{bytes .MemTotal}}</div>{{template "minibar" pctbar .MemPercent}}</div>
<div class="tile"><div class="v">{{printf "%.2f" .Load1}}</div><div class="k">load · {{printf "%.2f" .Load5}} 5m · {{printf "%.2f" .Load15}} 15m</div></div>
{{/* Only what this machine has sensors for: a VM has none, and a tile
saying 0 °C would be a reading nobody took. */}}
{{if or .CPUTempC .CPUPowerW}}<div class="tile"><div class="v">{{if .CPUTempC}}{{printf "%.0f" .CPUTempC}} °C{{else}}—{{end}}</div><div class="k">CPU{{if .CPUPowerW}} · {{printf "%.0f" .CPUPowerW}} W{{end}}</div></div>{{end}}
{{if .InletTempC}}<div class="tile"><div class="v">{{printf "%.1f" .InletTempC}} °C</div><div class="k">inlet air</div></div>{{end}}
{{else}}
<div class="tile"><div class="v">—</div><div class="k">CPU and RAM not measured lately</div></div>
{{end}}
@@ -29,6 +33,17 @@
</div>
{{if .Warnings}}{{range .Warnings}}<p class="note bad">{{.}}</p>{{end}}{{end}}
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{with .Load}}{{if .GPUs}}
<h2>GPUs</h2>
<table class="rows"><tr><th class="num">GPU</th><th class="num">Temperature</th><th class="num">Memory</th><th>Power</th><th>Held back by</th></tr>
{{range .GPUs}}<tr>
<td class="num">{{.Index}}</td>
<td class="num">{{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}}</td>
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}</td>
</tr>{{end}}</table>
{{end}}{{end}}
{{end}}
{{if or .Node.IBPorts .Node.Fabrics}}
+7
View File
@@ -109,6 +109,13 @@ var funcs = template.FuncMap{
return p
},
"ago": ago,
// percent is part of whole, 0 to 100, for a bar.
"percent": func(part, whole float64) float64 {
if whole <= 0 {
return 0
}
return part * 100 / whole
},
// shortNode is a node's name without what every node's name starts with.
"shortNode": strings.TrimPrefix,
"age": age,