Facility-aware placement: racks, PDUs, holds and power caps

Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:17:49 +02:00
1 parent d41104a452
commit 0d3eb82b9b
42 files changed
+3510 -63

No files matched your search

+28 -3
View File
@@ -139,6 +139,8 @@ func newCtlCmd() *cobra.Command {
newReplicasCmd(client, &asJSON),
newExplainCmd(client, &asJSON),
newFabricTestCmd(client, &asJSON),
newFacilityCmd(client, &asJSON),
newLabelCmd(client, &asJSON),
newNodeStateCmd(client, "cordon", "cordoned",
"Stop placing new work on a node; what runs there keeps running"),
newNodeStateCmd(client, "uncordon", "ready",
@@ -205,10 +207,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
fmt.Fprintln(w, "NODE\tSTATE\tLOCATION\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
for _, n := range nodes {
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
n.ID, stateColumn(n), locationColumn(n), n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
// What the node can run, which is the first thing to look
// at when a model will not place and the hardware looks
@@ -239,6 +241,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
fmt.Printf("\n%s: %s", n.ID, w)
said = true
}
if n.Held != "" {
fmt.Printf("\n%s: %s (kept out of placement)", n.ID, n.Held)
said = true
}
}
if said {
fmt.Println()
@@ -251,9 +257,28 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)")
c.Flags().StringArrayVar(&f.labels, "label", nil,
"only nodes carrying key=value; repeatable, and key alone matches any value")
c.Flags().StringVar(&f.domain, "domain", "", "only nodes in this rack, row, room, site or PDU, as level:name (rack:r12-07)")
return c
}
// stateColumn is a node's state, with "held" when a held domain keeps it out
// of placement while its own state says ready.
func stateColumn(n adminapi.NodeView) string {
if n.Held != "" && n.State == "ready" {
return "ready, held"
}
return n.State
}
// locationColumn is where a node is, in a few characters; a dash when it
// carries no location label.
func locationColumn(n adminapi.NodeView) string {
if n.Location == nil || n.Location.Empty() {
return "-"
}
return n.Location.Short()
}
// modelStatus says what is happening to a model, in the words that decide what
// to do about it.
//
+212
View File
@@ -0,0 +1,212 @@
package cli
import (
"fmt"
"io"
"maps"
"net/http"
"net/url"
"slices"
"strings"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
// newFacilityCmd shows the failure domains and holds them for maintenance.
func newFacilityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
cmd := &cobra.Command{
Use: "facility",
Short: "Racks, PDUs and rooms: their power against their caps, and what is held",
Long: `Where nodes are comes from their labels: llmgw.io/site, room, row, rack
and pdu, set in agent.yaml or with 'llmgw ctl label'. Placement spreads a
model's replicas across racks and PDUs, places nothing in a held domain, and
refuses a replica that would take a domain past its power cap (facility.budgets
in server.yaml).
USED is what counts against the cap: node by node, the larger of what the
replicas placed there are estimated to draw and what the node's sensors read.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var v adminapi.FacilityView
if err := client().get(cmd.Context(), "/api/v1/facility", &v); err != nil {
return err
}
if *asJSON {
return emitJSON(v)
}
printFacility(cmd.OutOrStdout(), v)
return nil
},
}
cmd.AddCommand(newFacilityHoldCmd(client), newFacilityReleaseCmd(client))
return cmd
}
// printFacility is the facility view as a table, then what is held, then
// the nodes nobody has said the place of.
func printFacility(out io.Writer, v adminapi.FacilityView) {
if len(v.Domains) == 0 {
fmt.Fprintln(out, "No node says where it is. Label them with llmgw.io/rack (and site, room, row, pdu):\n"+
" llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b")
return
}
w := tabTo(out)
fmt.Fprintln(w, "DOMAIN\tNODES\tUSED\tLIMIT\tRESERVED\tMEASURED\tHELD")
for _, d := range v.Domains {
limit, used := "-", facility.KW(d.UsedWatts)
if d.LimitWatts > 0 {
limit = facility.KW(d.LimitWatts)
used += fmt.Sprintf(" (%.0f%%)", 100*d.UsedWatts/d.LimitWatts)
}
held := "-"
if d.Held != nil {
held = "held"
if d.Held.Reason != "" {
held += ": " + d.Held.Reason
}
if d.Held.Drain {
held += " (drained)"
}
}
fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\n", d.Domain, len(d.Nodes), used, limit,
facility.KW(d.ReservedWatts), facility.KW(d.MeasuredWatts), held)
}
_ = w.Flush()
for _, h := range v.Holds {
fmt.Fprintf(out, "\n%s held by %s since %s", h.Domain, or(h.By, "someone"), h.At.Local().Format("2006-01-02 15:04"))
}
if len(v.Holds) > 0 {
fmt.Fprintln(out)
}
if len(v.Unlocated) > 0 {
fmt.Fprintf(out, "\nNo location: %s. Each is placed as a rack of its own.\n", strings.Join(v.Unlocated, ", "))
}
}
func newFacilityHoldCmd(client func() *ctlClient) *cobra.Command {
var req adminapi.HoldRequest
c := &cobra.Command{
Use: "hold <level:name>",
Short: "Place nothing new in a rack, row, PDU or room; with --drain, move its work off",
Long: `For maintenance, a cooling fault, work on a PDU: like cordoning every node in
the domain, including nodes labelled into it later, and kept across restarts
when the control plane has a store. What runs there keeps running unless
--drain is given; then each of its nodes is drained, one replica at a time
with the replacement up elsewhere first.`,
Example: ` llmgw ctl facility hold rack:r12-07 --reason "PDU A replacement"
llmgw ctl facility hold pdu:r12-07-a --drain --reason "breaker work"`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
req.Domain = args[0]
if _, err := facility.ParseDomain(req.Domain); err != nil {
return err
}
var v adminapi.HoldView
if err := client().do(cmd.Context(), http.MethodPost, "/api/v1/facility/holds", req, &v); err != nil {
return err
}
fmt.Printf("%s is held: nothing new is placed in it.\n", v.Domain)
if len(v.Nodes) > 0 {
fmt.Printf("Draining %s; their replicas move one at a time.\n", strings.Join(v.Nodes, ", "))
}
fmt.Printf("Release it with: llmgw ctl facility release %s\n", v.Domain)
return nil
},
}
c.Flags().StringVar(&req.Reason, "reason", "", "why, as explain-placement and the node list will say")
c.Flags().BoolVar(&req.Drain, "drain", false, "also drain the domain's nodes")
return c
}
func newFacilityReleaseCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "release <level:name>",
Short: "Allow placement in a held domain again",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
d, err := facility.ParseDomain(args[0])
if err != nil {
return err
}
var v adminapi.HoldView
if err := client().delete(cmd.Context(), "/api/v1/facility/holds/"+url.PathEscape(string(d)), &v); err != nil {
return err
}
fmt.Printf("%s is released.\n", v.Domain)
if len(v.Nodes) > 0 {
fmt.Printf("Ready again: %s.\n", strings.Join(v.Nodes, ", "))
}
return nil
},
}
}
// newLabelCmd sets labels on a node through the control plane.
func newLabelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "label <node> key=value... [key-...]",
Short: "Set or remove a node's labels, over those in its agent.yaml",
Long: `Labels set here are kept by the control plane and win over the agent's own;
key- removes one, which puts the agent's value for the key back if it has
one. The node need not have joined yet.
The llmgw.io/ labels say where a node is, for facility-aware placement:
site, room, row, rack, and pdu (a comma separated list for a dual-fed rack).
Any other key is free for a profile's node_selector.`,
Example: ` llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b
llmgw ctl label gpu01 llmgw.io/row=12 llmgw.io/room=hall-b
llmgw ctl label gpu01 llmgw.io/rack-`,
Args: cobra.MinimumNArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
req, err := parseLabelArgs(args[1:])
if err != nil {
return err
}
var v adminapi.NodeLabelsView
if err := client().do(cmd.Context(), http.MethodPost,
"/api/v1/nodes/"+url.PathEscape(args[0])+"/labels", req, &v); err != nil {
return err
}
if *asJSON {
return emitJSON(v)
}
if v.Labels == nil {
fmt.Printf("%s has not joined yet; it takes these labels when it does.\n", v.ID)
}
keys := slices.Sorted(maps.Keys(v.OperatorLabels))
if len(keys) == 0 {
fmt.Printf("%s carries no labels set here, only its agent's.\n", v.ID)
}
for _, k := range keys {
fmt.Printf("%s: %s=%s\n", v.ID, k, v.OperatorLabels[k])
}
if v.Location != nil {
fmt.Printf("%s is at %s.\n", v.ID, v.Location.Short())
}
return nil
},
}
}
// parseLabelArgs reads key=value to set and key- to remove.
func parseLabelArgs(args []string) (adminapi.NodeLabelsRequest, error) {
req := adminapi.NodeLabelsRequest{Set: map[string]string{}}
for _, a := range args {
if k, v, ok := strings.Cut(a, "="); ok {
if k == "" || v == "" {
return req, fmt.Errorf("%q: give key=value, or key- to remove a label", a)
}
req.Set[k] = v
continue
}
if k, ok := strings.CutSuffix(a, "-"); ok && k != "" {
req.Remove = append(req.Remove, k)
continue
}
return req, fmt.Errorf("%q: give key=value, or key- to remove a label", a)
}
return req, nil
}
+75
View File
@@ -0,0 +1,75 @@
package cli
import (
"bytes"
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
func TestLabelArgumentsSetWithEqualsAndRemoveWithADash(t *testing.T) {
req, err := parseLabelArgs([]string{"llmgw.io/rack=r1", "llmgw.io/pdu=p1,p2", "role-"})
if err != nil {
t.Fatal(err)
}
if req.Set["llmgw.io/rack"] != "r1" || req.Set["llmgw.io/pdu"] != "p1,p2" || !slices.Equal(req.Remove, []string{"role"}) {
t.Errorf("got %+v", req)
}
for _, bad := range []string{"rack", "=r1", "rack=", "-"} {
if _, err := parseLabelArgs([]string{bad}); err == nil {
t.Errorf("%q was accepted", bad)
}
}
}
func TestNodesAreFilteredByDomain(t *testing.T) {
nodes := fleet()
nodes[0].Location = &facility.Location{Rack: "r1", PDUs: []string{"p1"}}
nodes[1].Location = &facility.Location{Rack: "r2", PDUs: []string{"p1"}}
got, err := nodeFilter{domain: "pdu:p1"}.apply(nodes)
if err != nil || ids(got) != "n1,n2" {
t.Errorf("--domain pdu:p1 = %s, %v", ids(got), err)
}
if got, _ := (nodeFilter{domain: "rack:r2"}).apply(nodes); ids(got) != "n2" {
t.Errorf("--domain rack:r2 = %s", ids(got))
}
if _, err := (nodeFilter{domain: "r2"}).apply(nodes); err == nil {
t.Error("a domain with no level was accepted")
}
}
func TestTheLocationFormSetsWhatChangedAndRemovesWhatWasEmptied(t *testing.T) {
n := adminapi.NodeView{
Labels: map[string]string{"llmgw.io/rack": "r1", "llmgw.io/row": "12"},
OperatorLabels: map[string]string{"llmgw.io/row": "12"},
}
// site, room, row, rack, pdu
req := locationRequest(n, []string{"", "hall-b", "", "r1", "p1"})
if len(req.Set) != 2 || req.Set["llmgw.io/room"] != "hall-b" || req.Set["llmgw.io/pdu"] != "p1" {
t.Errorf("set %v: the unchanged rack is left alone", req.Set)
}
if !slices.Equal(req.Remove, []string{"llmgw.io/row"}) {
t.Errorf("remove %v, want the row the operator had set", req.Remove)
}
}
func TestTheFacilityTableShowsUseAgainstTheLimitAndWhatIsHeld(t *testing.T) {
var b bytes.Buffer
printFacility(&b, adminapi.FacilityView{
Domains: []adminapi.DomainView{
{Domain: "rack:r1", Nodes: []string{"a"}, LimitWatts: 10000, UsedWatts: 7500, ReservedWatts: 7000, MeasuredWatts: 7500,
Held: &adminapi.HoldView{Domain: "rack:r1", Reason: "PDU swap"}},
},
Holds: []adminapi.HoldView{{Domain: "rack:r1", By: "ilker"}},
Unlocated: []string{"b"},
})
out := b.String()
for _, want := range []string{"7.5 kW (75%)", "10.0 kW", "held: PDU swap", "rack:r1 held by ilker", "No location: b"} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out)
}
}
}
+19 -1
View File
@@ -6,6 +6,7 @@ import (
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
// nodeFilter narrows a node listing.
@@ -20,6 +21,8 @@ type nodeFilter struct {
class string
state string
labels []string
// domain is level:name, rack:r12-07.
domain string
}
// empty reports whether this filter would narrow anything.
@@ -28,7 +31,7 @@ type nodeFilter struct {
// different situations with different next steps, and "no nodes" for both
// makes a mistyped fabric look like an outage.
func (f nodeFilter) empty() bool {
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0 && f.domain == ""
}
func (f nodeFilter) describe() string {
@@ -45,6 +48,9 @@ func (f nodeFilter) describe() string {
for _, l := range f.labels {
parts = append(parts, "label "+l)
}
if f.domain != "" {
parts = append(parts, "domain "+f.domain)
}
return strings.Join(parts, ", ")
}
@@ -62,11 +68,23 @@ func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) {
want[k] = v
}
var domain facility.Domain
if f.domain != "" {
d, err := facility.ParseDomain(f.domain)
if err != nil {
return nil, fmt.Errorf("--domain: %w", err)
}
domain = d
}
out := make([]adminapi.NodeView, 0, len(in))
for _, n := range in {
if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) {
continue
}
if domain != "" && (n.Location == nil || !slices.Contains(n.Location.Domains(), domain)) {
continue
}
if f.class != "" && !slices.Contains(n.Classes, f.class) {
continue
}
+6
View File
@@ -654,6 +654,10 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
if u, ok := sink.(control.UsageSource); ok {
ctl.SetUsageSource(u)
}
// Power caps and estimates; Validate has already refused a bad one.
if pol, err := cfg.Facility.Policy(); err == nil {
ctl.Facility().SetPolicy(pol)
}
// With a shared store, two control planes can see each other, so this one
// competes rather than assuming it is alone. Without one there is nobody
// to compete with and it places unconditionally, exactly as before.
@@ -676,6 +680,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
ctl.Reconciler().SetMeasurements(control.Measurements(st, log))
// The engine builds chosen per node on the cluster page.
ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log))
// Domains held for maintenance and labels set on nodes.
ctl.Facility().UseStore(ctx, st)
log.Info("competing to be the control plane that places work",
"owner", lead.Owner())
}
+71
View File
@@ -17,6 +17,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
@@ -464,6 +465,7 @@ func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
return nil
}})
}
items = append(items, m.locationItems(n)...)
if n.GPUTotal > 0 {
for _, engine := range []string{"vllm", "sglang"} {
items = append(items, menuItem{engine + " build…", func(m *consoleModel) tea.Cmd {
@@ -825,3 +827,72 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
}
return items
}
// locationItems are the node menu's facility entries: where the node is,
// and holding or releasing its rack.
func (m consoleModel) locationItems(n adminapi.NodeView) []menuItem {
path := "/api/v1/nodes/" + url.PathEscape(n.ID) + "/labels"
var loc facility.Location
if n.Location != nil {
loc = *n.Location
}
items := []menuItem{{"Location…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Where " + n.ID + " is",
note: "Placement spreads a model's replicas across racks and PDUs, and keeps out of held ones. " +
"Empty leaves a level unknown; a PDU list is comma separated, for a dual-fed rack.",
fields: []formField{
textF("site", loc.Site, "sto1"), textF("room", loc.Room, "hall-b"),
textF("row", loc.Row, "12"), textF("rack", loc.Rack, "r12-07"),
textF("pdu", strings.Join(loc.PDUs, ","), "r12-07-a,r12-07-b"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
req := locationRequest(n, v)
if len(req.Set) == 0 && len(req.Remove) == 0 {
return nil, fmt.Errorf("nothing changed")
}
return m.call("Location of "+n.ID, http.MethodPost, path, req,
func(json.RawMessage) string { return n.ID + "'s location is set." }), nil
}})
return nil
}}}
switch {
case n.HeldDomain != "":
d := n.HeldDomain
items = append(items, menuItem{"Release " + d, func(m *consoleModel) tea.Cmd {
return m.call("Release "+d, http.MethodDelete, "/api/v1/facility/holds/"+url.PathEscape(d), nil,
func(json.RawMessage) string { return d + " is released." })
}})
case loc.Rack != "":
d := string(facility.Of(facility.Rack, loc.Rack))
items = append(items, menuItem{"Hold rack " + loc.Rack + "…", func(m *consoleModel) tea.Cmd {
m.openForm(&formDialog{title: "Hold rack " + loc.Rack,
note: "Nothing new is placed in the rack. Drain also moves its replicas off, one at a time.",
fields: []formField{textF("reason", "", "PDU A replacement"),
choiceF("drain", []string{"no", "yes"}, "no")},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
return m.call("Hold "+d, http.MethodPost, "/api/v1/facility/holds",
adminapi.HoldRequest{Domain: d, Reason: v[0], Drain: v[1] == "yes"},
func(json.RawMessage) string { return d + " is held." }), nil
}})
return nil
}})
}
return items
}
// locationRequest turns the location form's values, site to pdu, into the
// labels to set and the ones to remove: a level emptied that was set through
// the control plane is removed, which leaves the agent's own value if any.
func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequest {
req := adminapi.NodeLabelsRequest{Set: map[string]string{}}
for i, lv := range facility.Levels {
k, v := lv.Label(), strings.TrimSpace(vals[i])
switch {
case v != "" && n.Labels[k] != v:
req.Set[k] = v
case v == "" && n.OperatorLabels[k] != "":
req.Remove = append(req.Remove, k)
}
}
return req
}
+15 -3
View File
@@ -1041,11 +1041,11 @@ func (m consoleModel) panel() (title, body, keys string) {
if n.Load != nil {
cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent)
}
state := n.State
state := stateColumn(n)
if state != "ready" {
state = sWarn.Render(state)
}
rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
rows = append(rows, []string{n.ID, state, locationColumn(n), cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo})
}
t := "Nodes"
@@ -1055,7 +1055,7 @@ func (m consoleModel) panel() (title, body, keys string) {
if m.inPanel {
keys = rowKeys
}
return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
return t, "\n" + table([]string{"NODE", "STATE", "WHERE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
rows, m.row, pw, m.inPanel), keys
case cGPUs:
if m.inPanel {
@@ -1299,6 +1299,18 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) }
b.WriteString("\n")
kv("state", n.State)
if n.Location != nil {
where := []string{}
for _, d := range n.Location.Domains() {
where = append(where, d.Human())
}
kv("location", strings.Join(where, " · "))
} else {
kv("location", sDim.Render("not labelled: Enter, then Location…"))
}
if n.Held != "" {
kv("held", sWarn.Render(n.Held))
}
kv("address", n.MgmtAddr)
kv("system", n.OS)
kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago")