diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index efb33a06..ecf53027 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1060,6 +1060,16 @@ type Heartbeat struct { // report", and the first would quietly empty a node's inventory. WeightsChanged bool `protobuf:"varint,5,opt,name=weights_changed,json=weightsChanged,proto3" json:"weights_changed,omitempty"` Weights []*LocalWeights `protobuf:"bytes,6,rep,name=weights,proto3" json:"weights,omitempty"` + // memory_total_mb for the same reason again: RAM is added to a virtual + // machine while it runs, or across a reboot the agent does not notice, and a + // figure read once at registration kept a node that had grown to 32 GB + // looking like the 7 GB machine it used to be. 0 means not read. + MemoryTotalMb int64 `protobuf:"varint,7,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"` + // engine_versions only when they changed, like weights. An engine installed + // after the agent started is otherwise invisible to placement until + // somebody restarts the agent — which is a step nobody knows to take. + EnginesChanged bool `protobuf:"varint,8,opt,name=engines_changed,json=enginesChanged,proto3" json:"engines_changed,omitempty"` + EngineVersions map[string]string `protobuf:"bytes,9,rep,name=engine_versions,json=engineVersions,proto3" json:"engine_versions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1136,6 +1146,27 @@ func (x *Heartbeat) GetWeights() []*LocalWeights { return nil } +func (x *Heartbeat) GetMemoryTotalMb() int64 { + if x != nil { + return x.MemoryTotalMb + } + return 0 +} + +func (x *Heartbeat) GetEnginesChanged() bool { + if x != nil { + return x.EnginesChanged + } + return false +} + +func (x *Heartbeat) GetEngineVersions() map[string]string { + if x != nil { + return x.EngineVersions + } + return nil +} + // ShardStatus reports a shard's lifecycle transition. type ShardStatus struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1894,14 +1925,20 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" + - "\x06shared\x18\x03 \x01(\bR\x06shared\"\xfc\x01\n" + + "\x06shared\x18\x03 \x01(\bR\x06shared\"\xe2\x03\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + "\fnew_problems\x18\x03 \x03(\tR\vnewProblems\x12+\n" + "\astorage\x18\x04 \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12'\n" + "\x0fweights_changed\x18\x05 \x01(\bR\x0eweightsChanged\x120\n" + - "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\xbb\x03\n" + + "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12&\n" + + "\x0fmemory_total_mb\x18\a \x01(\x03R\rmemoryTotalMb\x12'\n" + + "\x0fengines_changed\x18\b \x01(\bR\x0eenginesChanged\x12P\n" + + "\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x1aA\n" + + "\x13EngineVersionsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbb\x03\n" + "\vShardStatus\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" + "\n" + @@ -1998,7 +2035,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 21) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -2020,7 +2057,8 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*Ack)(nil), // 17: llmgw.v1.Ack nil, // 18: llmgw.v1.Register.LabelsEntry nil, // 19: llmgw.v1.Register.EngineVersionsEntry - nil, // 20: llmgw.v1.EngineProfile.EnvEntry + nil, // 20: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 21: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -2041,17 +2079,18 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State 4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 0, // 18: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 11, // 19: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 13, // 20: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 20, // 21: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 22: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 23: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 23, // [23:24] is the sub-list for method output_type - 22, // [22:23] is the sub-list for method input_type - 22, // [22:22] is the sub-list for extension type_name - 22, // [22:22] is the sub-list for extension extendee - 0, // [0:22] is the sub-list for field type_name + 20, // 18: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 0, // 19: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 11, // 20: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 13, // 21: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 21, // 22: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 23: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 24: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 24, // [24:25] is the sub-list for method output_type + 23, // [23:24] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -2077,7 +2116,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 20, + NumMessages: 21, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 78cab9e6..1a54e026 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -222,6 +222,18 @@ message Heartbeat { // report", and the first would quietly empty a node's inventory. bool weights_changed = 5; repeated LocalWeights weights = 6; + + // memory_total_mb for the same reason again: RAM is added to a virtual + // machine while it runs, or across a reboot the agent does not notice, and a + // figure read once at registration kept a node that had grown to 32 GB + // looking like the 7 GB machine it used to be. 0 means not read. + int64 memory_total_mb = 7; + + // engine_versions only when they changed, like weights. An engine installed + // after the agent started is otherwise invisible to placement until + // somebody restarts the agent — which is a step nobody knows to take. + bool engines_changed = 8; + map engine_versions = 9; } // ShardStatus reports a shard's lifecycle transition. diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh index 3b79b009..fdec40ac 100755 --- a/deploy/bootstrap/onboard-node.sh +++ b/deploy/bootstrap/onboard-node.sh @@ -29,9 +29,11 @@ usage: $0 --node ID --addr HOST --server URL [options] --fabric NAME:hca1,hca2 . Omit for a node with no fabric, which can host single-node replicas and will never be given a multi-node one. - --mgmt-addr ADDR address engines are reachable on (default: --addr) - --mgmt-if NAME interface NCCL bootstraps over (default eth0) - --llamacpp-bin P path to llama-server + --mgmt-addr ADDR address engines are reachable on (default: the node's + address on the route to --server, found by the agent) + --mgmt-if NAME interface NCCL bootstraps over (default: the one holding + that address) + --llamacpp-bin P path to llama-server (default: PATH, then /opt/llama.cpp) --vllm-bin P path to the vllm entry point --sglang-python P python interpreter of the SGLang virtualenv --nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized @@ -47,7 +49,7 @@ USAGE exit 2 } -NODE= ADDR= SERVER= MGMT= MGMT_IF=eth0 +NODE= ADDR= SERVER= MGMT_OVERRIDE= MGMT_IF_OVERRIDE= NVIDIA_SMI= IB_ROOT=/sys/class/infiniband LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON= SSH_WAIT=180 @@ -60,8 +62,8 @@ while [ $# -gt 0 ]; do --addr) ADDR="$2"; shift 2;; --server) SERVER="$2"; shift 2;; --fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;; - --mgmt-addr) MGMT="$2"; shift 2;; - --mgmt-if) MGMT_IF="$2"; shift 2;; + --mgmt-addr) MGMT_OVERRIDE="$2"; shift 2;; + --mgmt-if) MGMT_IF_OVERRIDE="$2"; shift 2;; --llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;; --vllm-bin) VLLM_BIN="$2"; shift 2;; --sglang-python) SGLANG_PYTHON="$2"; shift 2;; @@ -74,7 +76,6 @@ while [ $# -gt 0 ]; do esac done [ -n "$NODE" ] && [ -n "$ADDR" ] && [ -n "$SERVER" ] || usage -MGMT="${MGMT:-$ADDR}" BINARY="${BINARY:-$(command -v llmgw)}" [ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; } [ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; } @@ -135,63 +136,49 @@ scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw" $SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw' say "writing the configuration" -FABRICS="fabrics: []" -if [ -n "$FABRIC_NAME" ]; then - FABRICS="fabrics: - - name: $FABRIC_NAME - hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]" -fi -cat > "$TMP/agent.yaml" </dev/null)" = "$NODE" ] && NODE_IS_HOSTNAME=yes +# Only what the node cannot find out for itself. Its name is its hostname, its +# address is the one on the route to the control plane, llama-server is looked +# for on PATH and where release archives are unpacked, GPUs are there when +# nvidia-smi is, and memory, disk and engines are re-read while it runs. Every +# setting written here is one that has to be kept true by hand — so an option +# below is written only when it was given, as an override. +{ + echo "# $NODE, written by onboard-node.sh." + echo "server: \"$SERVER\"" + echo + echo "tls:" + echo " ca: /etc/llmgw/ca.pem" + echo " cert: /etc/llmgw/agent.pem" + echo " key: /etc/llmgw/agent-key.pem" + # The certificate's common name is --node, and the control plane refuses a + # stream whose node id differs from it. Said explicitly only when the + # hostname would say something else. + [ "$NODE_IS_HOSTNAME" = yes ] || echo "node_id: $NODE" + [ -n "$MGMT_OVERRIDE" ] && echo "mgmt_addr: \"$MGMT_OVERRIDE\"" + [ -n "$MGMT_IF_OVERRIDE" ] && echo "mgmt_interface: $MGMT_IF_OVERRIDE" + if [ -n "$FABRIC_NAME" ]; then + # Fabric membership is the one thing that cannot be found: OpenSM's default + # subnet prefix is the same on every fabric, so a matching prefix proves + # nothing and a wrong guess hangs the first all-reduce. + echo "fabrics:" + echo " - name: $FABRIC_NAME" + echo " hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]" + fi + if [ -n "$NVIDIA_SMI" ] || [ "$IB_ROOT" != /sys/class/infiniband ]; then + echo "discovery:" + [ -n "$NVIDIA_SMI" ] && echo " nvidia_smi: \"$NVIDIA_SMI\"" + [ "$IB_ROOT" != /sys/class/infiniband ] && echo " ib_root: $IB_ROOT" + fi + if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then + echo "engines:" + [ -n "$LLAMACPP_BIN" ] && printf ' llamacpp:\n bin: "%s"\n' "$LLAMACPP_BIN" + [ -n "$VLLM_BIN" ] && printf ' vllm:\n bin: "%s"\n' "$VLLM_BIN" + [ -n "$SGLANG_PYTHON" ] && printf ' sglang:\n python: "%s"\n' "$SGLANG_PYTHON" + fi + true +} > "$TMP/agent.yaml" scp -q "$TMP/agent.yaml" "$SSH_USER@$ADDR:/tmp/agent.yaml" cat > "$TMP/llmgw-agent.service" <<'UNIT' diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml index 07f4ae19..5bd2ac50 100644 --- a/deploy/examples/agent.yaml +++ b/deploy/examples/agent.yaml @@ -4,12 +4,19 @@ # keep serving if the agent restarts: the router talks to them directly, and # orphan adoption picks them up again on the next start. -node_id: gpu-a-01 # defaults to the hostname +# Everything a node can find out about itself, it does: its id is its hostname, +# its address is the one on the route to the control plane, llama-server is +# looked for on PATH and in /opt/llama.cpp, GPUs are there when nvidia-smi is, +# weights are fetched into /var/lib/llmgw-weights and looked for there, and +# memory, disk and installed engines are re-read while the agent runs. So a +# plain CPU node needs only the two blocks below, and the same file on every +# node. What follows them are overrides, for when the defaults are wrong. + server: "http://llmgw-server.internal:7443" # The agent's half of mutual TLS. The certificate's common name must be this -# node's id, because the control plane compares the two and refuses the stream -# when they disagree: +# node's id — its hostname, unless node_id says otherwise — because the control +# plane compares the two and refuses the stream when they disagree: # # llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki # @@ -22,13 +29,19 @@ server: "http://llmgw-server.internal:7443" # cert: /etc/llmgw/agent.pem # key: /etc/llmgw/agent-key.pem +# ---- overrides --------------------------------------------------------------- + +# node_id: gpu-a-01 # when it is not the hostname + # The address engines are reachable on, and the address NCCL bootstraps over. # Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA. -mgmt_addr: "10.0.0.11" -mgmt_interface: eth0 +# mgmt_addr: "10.0.0.11" +# mgmt_interface: eth0 -labels: - rack: r12 +# Free-form, for a profile's node_selector. Nothing needs them; do not restate +# what is measured (memory, GPUs, engines), because a label goes stale. +# labels: +# rack: r12 heartbeat_interval: 5s reconnect_base: 1s @@ -55,12 +68,9 @@ discovery: # tensor-parallel group off the PCIe path inside a node. It needs a binary # built with `-tags nvml`; otherwise this silently falls back to nvidia-smi. prefer_nvml: true - skip_gpus: false -# Where this node keeps model weights. -# -# Unset by default, and unset means the node is not asked to look — which is -# not the same as having nothing, and constrains nothing. Storage differs at +# Where this node keeps model weights besides /var/lib/llmgw-weights, which is +# where fetched weights land and is always looked in. Storage differs at # every installation and llmgw deliberately does not own it: a Lustre mount, an # image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy # made by hand all look identical from here. diff --git a/docs/reference.md b/docs/reference.md index 0220e05a..57dd1b5f 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -330,6 +330,9 @@ into an outage is worse. `string` +NodeID defaults to the hostname. It must equal the common name of the +agent's certificate, so set it only where the two differ. + ### `server` `string` @@ -352,19 +355,30 @@ edit to every node's configuration. `string` +Hostname defaults to the machine's. + ### `mgmt_addr` `string` +MgmtAddr is the address engines are reached on. It defaults to this +machine's address on the route to the control plane, which is the one +the control plane and the router can reach. + ### `labels` `map[string]string` +Labels are free-form facts about a node for a profile's node_selector. +Nothing needs them: memory, GPUs, engines and disk are measured, and a +label restating one of those goes stale the day the machine changes. + ### `mgmt_interface` `string` -MgmtInterface is the ethernet interface NCCL bootstraps over. +MgmtInterface is the ethernet interface NCCL bootstraps over. It +defaults to the interface holding MgmtAddr, then eth0. ### `fabrics` @@ -411,10 +425,8 @@ Takes a [Config](#config) block. `[]string` -ModelDirs are the directories this node keeps model weights in. - -Empty by default, and empty means the node is not asked to look — which -is not the same as having nothing. Storage differs at every installation, +ModelDirs are the directories this node keeps model weights in, beyond +WeightsCache, which is always looked in. Storage differs at every installation, so llmgw observes it rather than owning it: a Lustre mount, an image exported over the fabric, local NVMe and a copy made by hand all look identical from here, and placement only needs to know whether the file @@ -426,7 +438,8 @@ is present. WeightsCache is where a remote source is downloaded to. Defaults to the first entry of ModelDirs, because a node that says where its models live -has already answered this question. +has already answered this question, and otherwise to +/var/lib/llmgw-weights. Fetching is the simple arrangement: every node keeps its own copy. The exotic ones need no code — weights on a shared filer or an image exported @@ -576,6 +589,8 @@ know that this page cannot guess. Bin is the llama-server binary. It is the server, not the CLI: the gateway talks OpenAI over HTTP and needs the one that serves it. +Empty looks on PATH, then where release archives are unpacked +(/opt/llama.cpp). ### `parallel` @@ -761,7 +776,8 @@ source of NVLink topology. `bool` -SkipGPUs omits GPU discovery, for nodes that have none. +SkipGPUs omits GPU discovery. A node without nvidia-smi is already +found to have none, so this is for one whose driver is broken. ## Prefix diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 32acfd62..730738e6 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -10,9 +10,11 @@ import ( "fmt" "io" "log/slog" + "maps" "math/rand/v2" "net" "net/http" + "net/url" "os" "path/filepath" "slices" @@ -35,6 +37,8 @@ import ( // Config configures the agent. type Config struct { + // NodeID defaults to the hostname. It must equal the common name of the + // agent's certificate, so set it only where the two differ. NodeID string `koanf:"node_id"` // Server is the control plane, and Servers is the same thing when there is // more than one. Both are accepted: every configuration written before @@ -45,13 +49,21 @@ type Config struct { // leader accepts an agent, so a standby is skipped in the time it takes to // be told so — which is what makes failover a reconnect rather than an // edit to every node's configuration. - Server string `koanf:"server"` - Servers []string `koanf:"servers"` - Hostname string `koanf:"hostname"` - MgmtAddr string `koanf:"mgmt_addr"` - Labels map[string]string `koanf:"labels"` + Server string `koanf:"server"` + Servers []string `koanf:"servers"` + // Hostname defaults to the machine's. + Hostname string `koanf:"hostname"` + // MgmtAddr is the address engines are reached on. It defaults to this + // machine's address on the route to the control plane, which is the one + // the control plane and the router can reach. + MgmtAddr string `koanf:"mgmt_addr"` + // Labels are free-form facts about a node for a profile's node_selector. + // Nothing needs them: memory, GPUs, engines and disk are measured, and a + // label restating one of those goes stale the day the machine changes. + Labels map[string]string `koanf:"labels"` - // MgmtInterface is the ethernet interface NCCL bootstraps over. + // MgmtInterface is the ethernet interface NCCL bootstraps over. It + // defaults to the interface holding MgmtAddr, then eth0. MgmtInterface string `koanf:"mgmt_interface"` // Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's // default subnet prefix is the same on every fabric, so nothing can be @@ -74,10 +86,8 @@ type Config struct { // keeps the plaintext behaviour, which only belongs on a trusted network. TLS tlsconf.Config `koanf:"tls"` - // ModelDirs are the directories this node keeps model weights in. - // - // Empty by default, and empty means the node is not asked to look — which - // is not the same as having nothing. Storage differs at every installation, + // ModelDirs are the directories this node keeps model weights in, beyond + // WeightsCache, which is always looked in. Storage differs at every installation, // so llmgw observes it rather than owning it: a Lustre mount, an image // exported over the fabric, local NVMe and a copy made by hand all look // identical from here, and placement only needs to know whether the file @@ -86,7 +96,8 @@ type Config struct { // WeightsCache is where a remote source is downloaded to. Defaults to the // first entry of ModelDirs, because a node that says where its models live - // has already answered this question. + // has already answered this question, and otherwise to + // /var/lib/llmgw-weights. // // Fetching is the simple arrangement: every node keeps its own copy. The // exotic ones need no code — weights on a shared filer or an image exported @@ -116,18 +127,47 @@ func DefaultConfig() Config { func (c *Config) normalize() error { d := DefaultConfig() - if c.NodeID == "" { - return errors.New("node_id is required") - } if err := c.TLS.Validate(); err != nil { return err } if err := c.normalizeServerURL(); err != nil { return err } + // Everything below is found rather than written. A file that has to say + // which machine it is on, and at what address, is a file that differs on + // every node — and a copy taken from one node to the next registers the + // second as the first. + host, _ := os.Hostname() + if c.NodeID == "" { + c.NodeID = host + } + if c.NodeID == "" { + return errors.New("node_id is not set and the hostname could not be read") + } + if c.Hostname == "" { + c.Hostname = host + } + if c.MgmtAddr == "" { + c.MgmtAddr = addrToward(c.Server) + } + if c.MgmtInterface == "" { + c.MgmtInterface = interfaceOf(c.MgmtAddr) + } if c.MgmtInterface == "" { c.MgmtInterface = d.MgmtInterface } + // The directory downloads land in is always one that is looked in. When + // they were two settings, a node could fill its disk with fetched weights + // that placement never saw — which is what every node here was doing. + if c.WeightsCache == "" && len(c.ModelDirs) > 0 { + c.WeightsCache = c.ModelDirs[0] + } + if c.WeightsCache == "" { + c.WeightsCache = DefaultWeightsCache + } + if !slices.Contains(c.ModelDirs, c.WeightsCache) { + c.ModelDirs = append(c.ModelDirs, c.WeightsCache) + } if c.HeartbeatInterval <= 0 { c.HeartbeatInterval = d.HeartbeatInterval } @@ -180,6 +220,61 @@ func (c *Config) normalizeServerURL() error { return nil } +// DefaultWeightsCache is where fetched weights go when nothing says otherwise. +const DefaultWeightsCache = "/var/lib/llmgw-weights" + +// addrToward is this machine's address on the route to the control plane. +// +// The address engines are reached on is whichever one the control plane can +// reach, and the kernel already knows which that is: a UDP "connection" picks +// the source address for the route without sending a packet. Asking the +// interfaces instead would have to guess among docker bridges, VPNs and +// link-local addresses. +func addrToward(server string) string { + u, err := url.Parse(server) + if err != nil || u.Host == "" { + return "" + } + host := u.Hostname() + port := u.Port() + if port == "" { + port = "443" + } + conn, err := net.Dial("udp", net.JoinHostPort(host, port)) + if err != nil { + return "" + } + defer conn.Close() + if a, ok := conn.LocalAddr().(*net.UDPAddr); ok { + return a.IP.String() + } + return "" +} + +// interfaceOf names the interface that holds an address, or "" when none does. +func interfaceOf(addr string) string { + ip := net.ParseIP(addr) + if ip == nil { + return "" + } + ifaces, err := net.Interfaces() + if err != nil { + return "" + } + for _, ifc := range ifaces { + addrs, err := ifc.Addrs() + if err != nil { + continue + } + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) { + return ifc.Name + } + } + } + return "" +} + // normalizeOne applies the scheme rules to a single address. func normalizeOne(addr string, secure bool) (string, error) { switch { @@ -209,13 +304,22 @@ type Agent struct { reg *engines.Registry http connectHTTPClient - // engineOnce probes the installed engines exactly once. + // engineOnce probes the installed engines once before registering. // - // Once, because probing runs each engine's --version and vLLM's imports - // torch — seconds on a cold cache. A reconnect is not a reason to pay that - // again, and what is installed does not change while the process runs. + // Once, because probing SGLang imports it in a Python interpreter — seconds + // on a cold cache — and a reconnect is not a reason to pay that again. + // Later changes are found by the slower re-probe the heartbeat starts, not + // by registering again: an engine installed while the agent runs used to + // stay invisible until somebody thought to restart the agent. engineOnce sync.Once + // engineVers is guarded by mu once the first probe has run. engineVers map[string]string + // enginesSent is the fingerprint of the engines the control plane last + // heard about, from registration or a heartbeat. Guarded by mu. + enginesSent string + // lastEngineProbe and probingEngines pace the re-probe. Guarded by mu. + lastEngineProbe time.Time + probingEngines bool mu sync.Mutex inv discovery.Inventory @@ -668,6 +772,21 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { a.mu.Unlock() hb := &llmgwv1.Heartbeat{NodeId: a.cfg.NodeID} + + // Read every time; it is one small file. Memory is added to a running VM, + // and a node that grew from 7 GB to 32 GB has to be seen to have grown + // without anybody restarting anything. + hb.MemoryTotalMb = int64(discovery.MemoryTotalMB()) + + a.reprobeEngines() + a.mu.Lock() + if fp := enginesFingerprint(a.engineVers); a.engineVers != nil && fp != a.enginesSent { + a.enginesSent = fp + hb.EnginesChanged = true + hb.EngineVersions = maps.Clone(a.engineVers) + } + a.mu.Unlock() + for _, g := range inv.GPUs { hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB)) } @@ -1041,9 +1160,61 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { // never going to be asked to run. func (a *Agent) engines(ctx context.Context) map[string]string { a.engineOnce.Do(func() { - a.engineVers = a.reg.Probe(ctx) + vers := a.reg.Probe(ctx) + a.mu.Lock() + a.engineVers, a.lastEngineProbe = vers, time.Now() + a.mu.Unlock() a.log.Info("engines available on this node", - "engines", engines.ProbedNames(a.engineVers)) + "engines", engines.ProbedNames(vers)) }) - return a.engineVers + a.mu.Lock() + defer a.mu.Unlock() + // Whatever registration sends is what the control plane now knows, so a + // heartbeat repeats it only once it differs. + a.enginesSent = enginesFingerprint(a.engineVers) + return maps.Clone(a.engineVers) +} + +// engineReprobe is how often the installed engines are looked for again. +// +// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a +// minute's delay in noticing is invisible, while SGLang's probe starts a +// Python interpreter that imports it. +const engineReprobe = time.Minute + +// reprobeEngines looks for installed engines again when it is time, off the +// heartbeat goroutine so a slow probe never delays a heartbeat. +func (a *Agent) reprobeEngines() { + a.mu.Lock() + due := !a.probingEngines && !a.lastEngineProbe.IsZero() && + time.Since(a.lastEngineProbe) >= engineReprobe + if due { + a.probingEngines = true + } + a.mu.Unlock() + if !due { + return + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + vers := a.reg.Probe(ctx) + + a.mu.Lock() + changed := enginesFingerprint(vers) != enginesFingerprint(a.engineVers) + a.engineVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false + a.mu.Unlock() + if changed { + a.log.Info("engines available on this node changed", + "engines", engines.ProbedNames(vers)) + } + }() +} + +func enginesFingerprint(vers map[string]string) string { + var b strings.Builder + for _, k := range slices.Sorted(maps.Keys(vers)) { + fmt.Fprintf(&b, "%s=%s\n", k, vers[k]) + } + return b.String() } diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 275cc31d..6e72ad26 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -24,7 +24,8 @@ type Options struct { // PreferNVML uses the NVML backend when this binary has one. It is the only // source of NVLink topology. PreferNVML bool `koanf:"prefer_nvml"` - // SkipGPUs omits GPU discovery, for nodes that have none. + // SkipGPUs omits GPU discovery. A node without nvidia-smi is already + // found to have none, so this is for one whose driver is broken. SkipGPUs bool `koanf:"skip_gpus"` // ModelDirs are scanned for weights. Empty means do not look. ModelDirs []string `koanf:"-"` diff --git a/internal/agent/discovery/gpu_smi.go b/internal/agent/discovery/gpu_smi.go index 18b43c63..ed011726 100644 --- a/internal/agent/discovery/gpu_smi.go +++ b/internal/agent/discovery/gpu_smi.go @@ -44,6 +44,13 @@ const mandatorySMIFields = 10 // NVML build adds. func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) { if binary == "" { + // Not configured and not installed is a node without an NVIDIA card, + // not a failed discovery. Reported as an error it also cut discovery + // short before InfiniBand was looked at, on every CPU node that had + // not been told by hand to skip GPUs. + if _, err := exec.LookPath("nvidia-smi"); err != nil { + return nil, nil + } binary = "nvidia-smi" } out, err := exec.CommandContext(ctx, binary, diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 775b77e2..56b2b1bf 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -3,6 +3,7 @@ package engines import ( "fmt" "os" + "os/exec" "path/filepath" "sort" "strconv" @@ -13,6 +14,8 @@ import ( type LlamaCppConfig struct { // Bin is the llama-server binary. It is the server, not the CLI: the // gateway talks OpenAI over HTTP and needs the one that serves it. + // Empty looks on PATH, then where release archives are unpacked + // (/opt/llama.cpp). Bin string `koanf:"bin"` // Parallel is how many requests a server handles at once, when a profile // does not say. llama.cpp defaults to one, which would make every second @@ -68,10 +71,7 @@ func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error { // Commands returns the processes for one shard, which is always one process. func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { - bin := l.cfg.Bin - if bin == "" { - bin = "llama-server" - } + bin := l.bin() p := s.Profile model, err := ggufIn(p.ModelPath) @@ -169,3 +169,34 @@ func ggufIn(path string) (string, error) { sort.Strings(names) return filepath.Join(path, names[0]), nil } + +// llamaServerPlaces are where llama.cpp's release archives are unpacked when +// they are not on PATH. The archive carries its shared libraries beside the +// binary, so it is usually left in one directory rather than installed. +var llamaServerPlaces = []string{ + "/opt/llama.cpp/llama-server", + "/opt/llama.cpp/bin/llama-server", + "/opt/llama.cpp/build/bin/llama-server", + "/usr/local/llama.cpp/llama-server", +} + +// bin is the llama-server to run: the configured one, else the one on PATH, +// else the first of the usual unpack locations that exists. +// +// Found rather than required in the configuration, because every node here +// had the same path written into its file by hand, and a node set up without +// it reported no engine at all. +func (l *LlamaCpp) bin() string { + if l.cfg.Bin != "" { + return l.cfg.Bin + } + if p, err := exec.LookPath("llama-server"); err == nil { + return p + } + for _, p := range llamaServerPlaces { + if info, err := os.Stat(p); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + return p + } + } + return "llama-server" +} diff --git a/internal/agent/engines/probe.go b/internal/agent/engines/probe.go index 7e9819a2..51edd809 100644 --- a/internal/agent/engines/probe.go +++ b/internal/agent/engines/probe.go @@ -159,11 +159,7 @@ func firstVersion(s string) string { // Probe reports whether the llama.cpp server is installed. func (l *LlamaCpp) Probe(context.Context) (string, error) { - bin := l.cfg.Bin - if bin == "" { - bin = "llama-server" - } - return installedBinary(bin) + return installedBinary(l.bin()) } // Probe reports whether vLLM is installed. diff --git a/internal/agent/selfdescribe_test.go b/internal/agent/selfdescribe_test.go new file mode 100644 index 00000000..41afc719 --- /dev/null +++ b/internal/agent/selfdescribe_test.go @@ -0,0 +1,80 @@ +package agent + +import ( + "os" + "slices" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" +) + +func TestAnAgentConfigNeedsOnlyWhereTheControlPlaneIs(t *testing.T) { + // Every node carried its own name, its own address and the same two + // directories written out by hand, so no two agent files were alike and a + // copy taken from one node registered the next as the first. + cfg := Config{Server: "127.0.0.1:7443"} + if err := cfg.normalize(); err != nil { + t.Fatal(err) + } + host, _ := os.Hostname() + if cfg.NodeID != host || cfg.Hostname != host { + t.Errorf("node_id %q, hostname %q; expected the machine's name %q", + cfg.NodeID, cfg.Hostname, host) + } + if cfg.MgmtAddr != "127.0.0.1" { + t.Errorf("mgmt_addr %q, expected the address on the route to the control plane", + cfg.MgmtAddr) + } + if cfg.WeightsCache != DefaultWeightsCache { + t.Errorf("weights land in %q", cfg.WeightsCache) + } + if !slices.Contains(cfg.ModelDirs, cfg.WeightsCache) { + t.Errorf("the download directory %q is not looked in: %v", + cfg.WeightsCache, cfg.ModelDirs) + } +} + +func TestWhatIsWrittenStillWins(t *testing.T) { + cfg := Config{ + Server: "127.0.0.1:7443", NodeID: "gpu-a-01", MgmtAddr: "10.9.9.9", + ModelDirs: []string{"/lustre/models"}, WeightsCache: "/scratch/w", + } + if err := cfg.normalize(); err != nil { + t.Fatal(err) + } + if cfg.NodeID != "gpu-a-01" || cfg.MgmtAddr != "10.9.9.9" { + t.Errorf("configured identity overridden: %q %q", cfg.NodeID, cfg.MgmtAddr) + } + if !slices.Equal(cfg.ModelDirs, []string{"/lustre/models", "/scratch/w"}) { + t.Errorf("model dirs %v", cfg.ModelDirs) + } +} + +func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) { + cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir()} + cfg.Superv.WorkDir = t.TempDir() + a, err := New(cfg, nil, nil) + if err != nil { + t.Fatal(err) + } + engines := func(m *llmgwv1.AgentMessage) *llmgwv1.Heartbeat { return m.GetHeartbeat() } + + a.mu.Lock() + a.engineVers = map[string]string{"llamacpp": "unknown"} + a.enginesSent = enginesFingerprint(a.engineVers) + a.mu.Unlock() + if hb := engines(a.heartbeat()); hb.GetEnginesChanged() { + t.Errorf("unchanged engines were resent") + } + + a.mu.Lock() + a.engineVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"} + a.mu.Unlock() + hb := engines(a.heartbeat()) + if !hb.GetEnginesChanged() || len(hb.GetEngineVersions()) != 2 { + t.Errorf("an installed engine was not reported: %v", hb.GetEngineVersions()) + } + if engines(a.heartbeat()).GetEnginesChanged() { + t.Errorf("reported again on the next heartbeat") + } +} diff --git a/internal/config/example_test.go b/internal/config/example_test.go index 11ae62bb..26ed1b0b 100644 --- a/internal/config/example_test.go +++ b/internal/config/example_test.go @@ -39,8 +39,10 @@ func TestTheDocumentedAgentExampleLoads(t *testing.T) { if f.RateGbps == 0 { t.Error("the fabric's rate did not bind") } - if cfg.NodeID == "" || cfg.Server == "" { - t.Errorf("node_id=%q server=%q", cfg.NodeID, cfg.Server) + // node_id is left to the hostname, so the example need not say it; where + // the control plane is, nothing can find out. + if cfg.Server == "" { + t.Errorf("server=%q", cfg.Server) } } diff --git a/internal/control/agentsvc/heartbeat_test.go b/internal/control/agentsvc/heartbeat_test.go index 255f5db2..97d57e2e 100644 --- a/internal/control/agentsvc/heartbeat_test.go +++ b/internal/control/agentsvc/heartbeat_test.go @@ -1,6 +1,7 @@ package agentsvc import ( + "log/slog" "testing" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" @@ -11,7 +12,8 @@ import ( func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) } func newTestService() *Service { - return &Service{inv: inventory.NewStore(), conns: map[string]*conn{}} + return &Service{inv: inventory.NewStore(), conns: map[string]*conn{}, + log: slog.New(slog.DiscardHandler)} } func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) { @@ -112,3 +114,44 @@ func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) { t.Errorf("disk changed: %d", free) } } + +func TestAHeartbeatCarriesMemoryAddedToARunningNode(t *testing.T) { + // Read once at registration, a node that was given 32 GB went on looking + // like the 7 GB machine it had been, and every model that needed the + // difference stayed unplaced until somebody restarted its agent. + s := newTestService() + nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024}) + + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 32 * 1024}) + + if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 { + t.Errorf("memory is %d MB, expected the 32 GB the node now has", n.MemoryTotalMB) + } + + // Zero is an agent that could not read it, not a node that lost its RAM. + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"}) + if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 { + t.Errorf("a heartbeat without memory erased it: %d", n.MemoryTotalMB) + } +} + +func TestAHeartbeatCarriesAnEngineInstalledAfterRegistration(t *testing.T) { + s := newTestService() + nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, + EngineVersions: map[string]string{"llamacpp": "unknown"}}) + + // Unchanged engines are not resent, and silence must not clear them. + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 1024}) + if n, _ := s.inv.Get("n1"); len(n.EngineVersions) != 1 { + t.Fatalf("engines changed without being reported: %v", n.EngineVersions) + } + + s.applyHeartbeat("n1", &llmgwv1.Heartbeat{ + NodeId: "n1", EnginesChanged: true, + EngineVersions: map[string]string{"llamacpp": "unknown", "vllm": "unknown"}, + }) + n, _ := s.inv.Get("n1") + if _, ok := n.EngineVersions["vllm"]; !ok { + t.Errorf("the newly installed engine is not known: %v", n.EngineVersions) + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 1507899b..e8678d81 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log/slog" + "maps" "net/http" "slices" "sync" @@ -269,7 +270,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { used := hb.GetGpuMemoryUsedMb() problems := hb.GetNewProblems() storage := hb.GetStorage() - if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() { + memMB := int(hb.GetMemoryTotalMb()) + if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && + !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() { return } @@ -316,6 +319,26 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { }) } } + + // RAM and engines change under a running agent too: memory added to a VM, + // an engine installed this afternoon. Said in the log when they do, because + // "the node now qualifies" is the answer to why a model placed an hour after + // it was asked for, and "the node no longer does" is the answer to why one + // stopped placing. + if memMB > 0 && memMB != n.MemoryTotalMB { + s.log.Info("node memory changed", "node", nodeID, + "from_mb", n.MemoryTotalMB, "to_mb", memMB) + n.MemoryTotalMB = memMB + } + if hb.GetEnginesChanged() { + vers := maps.Clone(hb.GetEngineVersions()) + if !maps.Equal(vers, n.EngineVersions) { + s.log.Info("node engines changed", "node", nodeID, + "from", slices.Sorted(maps.Keys(n.EngineVersions)), + "to", slices.Sorted(maps.Keys(vers))) + } + n.EngineVersions = vers + } s.inv.Upsert(n) }