The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7bfe70c8df
commit
c866b3a8ee
34 files changed
+954
-52
No files matched your search
@@ -2219,12 +2219,27 @@ type GPUSensor struct {
|
||||
NvlinkReplayErrors int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"`
|
||||
NvlinkRecoveryErrors int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"`
|
||||
// xid_errors are the driver's error events since the agent started.
|
||||
XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
|
||||
LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
|
||||
LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
|
||||
LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
|
||||
LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
|
||||
LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
|
||||
LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
|
||||
// How much of the chip the work on it uses, which utilization_percent
|
||||
// does not say: that is the share of time any kernel ran, and a kernel on
|
||||
// one SM of a hundred counts in full. has_profile says these are present;
|
||||
// -1 is not reported. Read from NVML's GPM on Hopper and later, or from a
|
||||
// dcgm-exporter on the node; profile_source says which ("gpm", "dcgm").
|
||||
HasProfile bool `protobuf:"varint,29,opt,name=has_profile,json=hasProfile,proto3" json:"has_profile,omitempty"`
|
||||
// sm_active_percent is the share of time the SMs had work, averaged over
|
||||
// all of them; sm_occupancy_percent how full those SMs were with warps.
|
||||
SmActivePercent float64 `protobuf:"fixed64,30,opt,name=sm_active_percent,json=smActivePercent,proto3" json:"sm_active_percent,omitempty"`
|
||||
SmOccupancyPercent float64 `protobuf:"fixed64,31,opt,name=sm_occupancy_percent,json=smOccupancyPercent,proto3" json:"sm_occupancy_percent,omitempty"`
|
||||
// tensor_active_percent is the tensor cores; dram_active_percent the
|
||||
// memory interface, which decode is bound by.
|
||||
TensorActivePercent float64 `protobuf:"fixed64,32,opt,name=tensor_active_percent,json=tensorActivePercent,proto3" json:"tensor_active_percent,omitempty"`
|
||||
DramActivePercent float64 `protobuf:"fixed64,33,opt,name=dram_active_percent,json=dramActivePercent,proto3" json:"dram_active_percent,omitempty"`
|
||||
ProfileSource string `protobuf:"bytes,34,opt,name=profile_source,json=profileSource,proto3" json:"profile_source,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *GPUSensor) Reset() {
|
||||
@@ -2453,6 +2468,48 @@ func (x *GPUSensor) GetLastXidUnix() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetHasProfile() bool {
|
||||
if x != nil {
|
||||
return x.HasProfile
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetSmActivePercent() float64 {
|
||||
if x != nil {
|
||||
return x.SmActivePercent
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetSmOccupancyPercent() float64 {
|
||||
if x != nil {
|
||||
return x.SmOccupancyPercent
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetTensorActivePercent() float64 {
|
||||
if x != nil {
|
||||
return x.TensorActivePercent
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetDramActivePercent() float64 {
|
||||
if x != nil {
|
||||
return x.DramActivePercent
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetProfileSource() string {
|
||||
if x != nil {
|
||||
return x.ProfileSource
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
type ShardLoad struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -5377,7 +5434,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
" \x01(\x01R\tcpuPowerW\x12 \n" +
|
||||
"\finlet_temp_c\x18\v \x01(\x01R\n" +
|
||||
"inletTempC\x12'\n" +
|
||||
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" +
|
||||
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xe8\n" +
|
||||
"\n" +
|
||||
"\tGPUSensor\x12\x14\n" +
|
||||
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" +
|
||||
"\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" +
|
||||
@@ -5409,7 +5467,14 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" +
|
||||
"\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" +
|
||||
"\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" +
|
||||
"\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" +
|
||||
"\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\x12\x1f\n" +
|
||||
"\vhas_profile\x18\x1d \x01(\bR\n" +
|
||||
"hasProfile\x12*\n" +
|
||||
"\x11sm_active_percent\x18\x1e \x01(\x01R\x0fsmActivePercent\x120\n" +
|
||||
"\x14sm_occupancy_percent\x18\x1f \x01(\x01R\x12smOccupancyPercent\x122\n" +
|
||||
"\x15tensor_active_percent\x18 \x01(\x01R\x13tensorActivePercent\x12.\n" +
|
||||
"\x13dram_active_percent\x18! \x01(\x01R\x11dramActivePercent\x12%\n" +
|
||||
"\x0eprofile_source\x18\" \x01(\tR\rprofileSource\"m\n" +
|
||||
"\tShardLoad\x12\x19\n" +
|
||||
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
|
||||
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
|
||||
|
||||
@@ -448,6 +448,21 @@ message GPUSensor {
|
||||
int32 last_xid = 26;
|
||||
string last_xid_text = 27;
|
||||
int64 last_xid_unix = 28;
|
||||
// How much of the chip the work on it uses, which utilization_percent
|
||||
// does not say: that is the share of time any kernel ran, and a kernel on
|
||||
// one SM of a hundred counts in full. has_profile says these are present;
|
||||
// -1 is not reported. Read from NVML's GPM on Hopper and later, or from a
|
||||
// dcgm-exporter on the node; profile_source says which ("gpm", "dcgm").
|
||||
bool has_profile = 29;
|
||||
// sm_active_percent is the share of time the SMs had work, averaged over
|
||||
// all of them; sm_occupancy_percent how full those SMs were with warps.
|
||||
double sm_active_percent = 30;
|
||||
double sm_occupancy_percent = 31;
|
||||
// tensor_active_percent is the tensor cores; dram_active_percent the
|
||||
// memory interface, which decode is bound by.
|
||||
double tensor_active_percent = 32;
|
||||
double dram_active_percent = 33;
|
||||
string profile_source = 34;
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
|
||||
Reference in new issue
Block a user