{{define "node"}} llmgw — {{.Node.ID}}

{{.Node.ID}}

{{t (print "dash.state." .Node.State)}} {{with .Node.OS}} · {{.}}{{end}}{{with .Node.MgmtAddr}} · {{.}}{{end}} {{with .Node.AgentVersion}} · llmgw {{.}}{{end}} · {{if .Node.LastSeenAgo}}{{t "node.last_seen" .Node.LastSeenAgo}}{{else}}{{t "node.never_seen"}}{{end}} · {{t "dash.nodes.logs"}}

{{if .OnAdminListener}}{{else}}{{template "nav" .Nav}}{{end}}
{{/* Refreshed in place, like the cluster page. */}}
{{with .Node}}
{{with .Load}}
{{num .CPUPercent 0}}%
CPU{{if .CPUs}} · {{t "node.cores" .CPUs}}{{end}}
{{template "minibar" pctbar .CPUPercent}}
{{bytes .MemUsed}}
{{t "node.ram_in_use_of" (bytes .MemTotal)}}
{{template "minibar" pctbar .MemPercent}}
{{num .Load1 2}}
{{t "node.load" (num .Load5 2) (num .Load15 2)}}
{{/* Only what this machine has sensors for: a VM has none, and a tile saying 0 °C would be a reading nobody took. */}} {{if or .CPUTempC .CPUPowerW}}
{{if .CPUTempC}}{{num .CPUTempC 0}} °C{{else}}—{{end}}
CPU{{if .CPUPowerW}} · {{num .CPUPowerW 0}} W{{end}}
{{end}} {{if .InletTempC}}
{{num .InletTempC 1}} °C
{{t "node.inlet"}}
{{end}} {{else}}
—
{{t "node.not_measured"}}
{{end}} {{if .DiskTotalGB}}
{{.DiskFreeGB}}G
{{t "node.disk_free" .DiskTotalGB}}
{{end}} {{if .GPUTotal}}
{{.GPUFree}}/{{.GPUTotal}}
{{t "node.gpus_free"}}
{{end}}
{{/* The node's warnings and what is unhealthy come from the control plane and the agent, in English. */}} {{if .Warnings}}{{range .Warnings}}

{{.}}

{{end}}{{end}} {{if .Unhealthy}}

{{join .Unhealthy}}

{{end}} {{with .Load}}{{if .GPUs}}

{{t "node.gpus"}}

{{range .GPUs}} {{/* From the last sample; a dash where the driver or an older agent did not say, rather than a zero nobody measured. */}} {{/* The driver's reasons and the wear are named as the driver and the inventory name them, in English. */}} {{end}}
GPU{{t "node.gpu.busy"}}{{t "node.gpu.memory_used"}}{{t "node.gpu.temp"}}{{t "node.gpu.memory_temp"}}{{t "node.gpu.power"}}{{t "node.gpu.clock"}}{{t "node.gpu.models"}}{{t "node.gpu.held_back"}}
{{.Index}}{{if and .MemoryTotalMB (ge .UtilPercent 0.0)}}{{template "minibar" pctbar .UtilPercent}}{{num .UtilPercent 0}}%{{if ge .MemUtilPercent 0.0}} {{t "node.gpu.memory_util" (num .MemUtilPercent 0)}}{{end}}{{else}}—{{end}} {{if .MemoryTotalMB}}{{template "minibar" pctbar (percent .MemoryUsedMB .MemoryTotalMB)}}{{num (gib .MemoryUsedMB) 1}} {{t "node.of_gb" (num (gib .MemoryTotalMB) 0)}}{{else}}—{{end}} {{if .TempC}}{{num .TempC 0}} °C{{else}}—{{end}} {{if .MemoryTempC}}{{num .MemoryTempC 0}} °C{{else}}—{{end}} {{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}{{num .PowerW 0}} W {{t "node.of" (num .PowerLimitW 0)}}{{else if .PowerW}}{{num .PowerW 0}} W{{else}}—{{end}} {{if .SMClockMHz}}{{num .SMClockMHz 0}} MHz{{else}}—{{end}}{{with index $.GPUModels .Index}}{{join .}}{{else}}—{{end}} {{if .Throttle}}{{join .Throttle}}{{else}}{{t "node.gpu.nothing"}}{{end}}{{if gt .ECCUncorrected 0.0}} {{t "node.gpu.ecc" (num .ECCUncorrected 0)}}{{end}}{{with .Wear}}{{with .Problems}}
{{join .}}{{end}}{{if gt .PCIeReplays 0}} {{t "node.gpu.pcie" (count .PCIeReplays)}}{{end}}{{end}}
{{end}}{{end}} {{end}} {{if or .Node.IBPorts .Node.Fabrics}}

{{t "node.fabric"}}

{{if .Node.Fabrics}}{{t "node.fabric.on" (join .Node.Fabrics)}} · {{t "node.fabric.map"}}{{else}}{{t "node.fabric.none"}}{{end}}

{{if .Node.IBPorts}} {{range .Node.IBPorts}} {{/* The port's state and its counters are the adapter's own words. */}} {{end}}
{{t "node.ib.adapter"}}{{t "dash.col.state"}}{{t "node.ib.rate"}}{{t "node.fabric"}}{{t "node.ib.physically"}}{{t "node.ib.errors"}}{{t "node.ib.sent"}}{{t "node.ib.received"}}
{{.HCA}}/{{.Port}}{{if eq .LinkLayer "Ethernet"}} RoCE{{end}}{{.State}}{{with .PhysState}} {{.}}{{end}} {{if .RateGbps}}{{t "fabric.gbps" .RateGbps}}{{else}}—{{end}} {{or .Fabric "—"}} {{if .Physical}}{{.Physical}}{{else if .SMError}}{{t "node.unknown"}}{{else}}—{{end}} {{if .Errors}}{{range $k, $v := .Errors}}{{$k}} {{$v}} {{end}}{{else}}{{t "node.none"}}{{end}} {{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}} {{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}

{{t "node.ib.note"}}

{{end}} {{end}} {{if .Node.GPUs}}

{{t "node.nvl.title"}}

{{with .Node.NVLinkDomain}}{{th "node.nvl.domain" .}} {{else}}{{t "node.nvl.ends_here"}}{{end}} {{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}{{t "node.imex.active"}}, {{tn "node.imex.channels" .Node.IMEXChannels}}{{else}}{{with .Node.IMEX}}{{.}}{{else}}{{t "node.imex.not_installed"}}{{end}}{{end}}{{end}}

{{range .Node.GPUs}}{{end}} {{range $a := .Node.GPUs}} {{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}{{else if hasInt $a.NVLinkPeers $b.Index}}{{else}}{{end}}{{end}} {{end}}
{{.Index}}
GPU {{$a.Index}}NV·
{{range .Node.GPUs}} {{/* The bandwidth and fabric state are the driver's words. */}} {{end}}
GPU{{t "node.gpu.name"}}{{t "node.gpu.compute"}}{{t "node.gpu.numa"}}{{t "node.gpu.nearest"}}{{t "node.gpu.multinode"}}
{{.Index}}{{.Name}}{{.ComputeCapability}}{{.NUMA}} {{if .NearestNIC}}{{.NearestNIC}} {{.NICPath}}{{else}}—{{end}}{{if .Domain}}{{t "node.gpu.joined"}}{{if and .Bandwidth (ne .Bandwidth "Full")}} {{t "node.gpu.bandwidth" .Bandwidth}}{{end}}{{else if .FabricState}}{{.FabricState}}{{else}}—{{end}}

{{t "node.nvl.note"}}

{{end}}

{{t "node.runs"}}

{{if .Engines}} {{range .Engines}}{{end}}
{{t "dash.col.model"}}{{t "dash.col.replica"}}{{t "dash.col.state"}}GPUCPU{{t "node.memory"}}
{{.Model}} {{.Replica}} {{t (print "dash.state." .State)}} {{if .GPUs}}{{range $i, $g := .GPUs}}{{if $i}}, {{end}}{{$g}}{{end}}{{else}}—{{end}} {{template "minibar" pctbar .CPUShare}}{{t "node.cores_f" (num .CPUCores 1)}} {{if .MemoryBytes}}{{bytes .MemoryBytes}}{{else}}—{{end}}
{{else}}

{{t "node.no_engines"}}

{{end}}

{{if .EngineVersions}}{{t "node.engines" (join .EngineVersions)}}{{else if .Node.Engines}}{{t "node.engines" (join .Node.Engines)}}{{else}}{{t "node.engines_none"}}{{end}}{{if .Node.Models}} · {{t "node.model_files" (len .Node.Models)}}{{end}}.

{{if .HasHistory}}{{with .Charts}}

{{t "node.last" .Window}}

{{if .Err}}

{{t "dash.no_history" .Err}}

{{else}}
{{template "tschart" dict "t" .CPU "title" "CPU" "note" (t "node.chart.cpu_note")}} {{template "tschart" dict "t" .Memory "title" (t "node.chart.ram") "note" (t "node.chart.ram_note")}}
{{template "tschart" dict "t" .EngineCPU "title" (t "node.chart.cores") "note" (t "node.chart.cores_note")}} {{template "tschart" dict "t" .EngineMemory "title" (t "node.chart.engine_memory") "note" (t "node.chart.engine_memory_note")}}
{{if $.Node.GPUTotal}}
{{template "tschart" dict "t" .GPUUtil "title" (t "node.chart.gpu_busy") "note" (t "node.chart.gpu_busy_note")}} {{template "tschart" dict "t" .GPUMemory "title" (t "node.chart.gpu_memory") "note" (t "node.chart.gpu_memory_note")}}
{{template "tschart" dict "t" .GPUTemp "title" (t "node.chart.gpu_temp") "note" (t "node.chart.gpu_temp_note")}} {{template "tschart" dict "t" .GPUPower "title" (t "node.chart.gpu_power") "note" (t "node.chart.gpu_power_note")}}
{{end}}
{{template "tschart" dict "t" .Load "title" (t "node.chart.load") "note" (t "node.chart.load_note")}} {{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "node.chart.queue_note")}}
{{end}} {{end}}{{else}}

{{th "node.no_store"}}

{{end}}
{{end}}