Let a node describe itself instead of being described by hand
Every agent file on the cluster named its own node, its own address, the path to llama-server and two weight directories — different on every node, kept true by hand, and not true: node05 carried "ram: 32g" beside a measured 31 GB, looked for weights in an empty directory while fetching them into another, and the other four never looked in theirs at all, so four nodes reported no disk. Growing a node's RAM would have gone unnoticed, because memory was read once, at registration. The agent now finds these out. Its id is the hostname, which is also the certificate's common name. Its address is the one on the route to the control plane, which the kernel picks for a UDP dial without sending anything. llama-server is looked for on PATH and then where release archives are unpacked. A missing nvidia-smi is a node without NVIDIA cards rather than a failed discovery — which also cut discovery short before InfiniBand on every CPU node not told to skip GPUs. Fetched weights land in /var/lib/llmgw-weights by default, and wherever they land is always looked in. Memory is read on every heartbeat, and installed engines are probed again each minute and sent when they change, so a VM given more RAM or a node given vLLM becomes a candidate without anyone restarting its agent. The control plane logs both changes, since "the node now qualifies" is the answer to why a model placed an hour after it was asked for. What is written still wins. The agent file shrinks to the control plane's addresses and the TLS files, the same on every node; onboarding writes only the overrides it was given. What cannot be found stays configuration: which fabric an adapter is on, because OpenSM's default prefix is the same on all of them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2cae92d113
commit
20deb22dfa
14 files changed
+552
-134
No files matched your search
@@ -1060,6 +1060,16 @@ type Heartbeat struct {
|
||||
// report", and the first would quietly empty a node's inventory.
|
||||
WeightsChanged bool `protobuf:"varint,5,opt,name=weights_changed,json=weightsChanged,proto3" json:"weights_changed,omitempty"`
|
||||
Weights []*LocalWeights `protobuf:"bytes,6,rep,name=weights,proto3" json:"weights,omitempty"`
|
||||
// memory_total_mb for the same reason again: RAM is added to a virtual
|
||||
// machine while it runs, or across a reboot the agent does not notice, and a
|
||||
// figure read once at registration kept a node that had grown to 32 GB
|
||||
// looking like the 7 GB machine it used to be. 0 means not read.
|
||||
MemoryTotalMb int64 `protobuf:"varint,7,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"`
|
||||
// engine_versions only when they changed, like weights. An engine installed
|
||||
// after the agent started is otherwise invisible to placement until
|
||||
// somebody restarts the agent — which is a step nobody knows to take.
|
||||
EnginesChanged bool `protobuf:"varint,8,opt,name=engines_changed,json=enginesChanged,proto3" json:"engines_changed,omitempty"`
|
||||
EngineVersions map[string]string `protobuf:"bytes,9,rep,name=engine_versions,json=engineVersions,proto3" json:"engine_versions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -1136,6 +1146,27 @@ func (x *Heartbeat) GetWeights() []*LocalWeights {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetMemoryTotalMb() int64 {
|
||||
if x != nil {
|
||||
return x.MemoryTotalMb
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetEnginesChanged() bool {
|
||||
if x != nil {
|
||||
return x.EnginesChanged
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetEngineVersions() map[string]string {
|
||||
if x != nil {
|
||||
return x.EngineVersions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShardStatus reports a shard's lifecycle transition.
|
||||
type ShardStatus struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -1894,14 +1925,20 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
|
||||
"\n" +
|
||||
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
|
||||
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xfc\x01\n" +
|
||||
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xe2\x03\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
"\fnew_problems\x18\x03 \x03(\tR\vnewProblems\x12+\n" +
|
||||
"\astorage\x18\x04 \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12'\n" +
|
||||
"\x0fweights_changed\x18\x05 \x01(\bR\x0eweightsChanged\x120\n" +
|
||||
"\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\xbb\x03\n" +
|
||||
"\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12&\n" +
|
||||
"\x0fmemory_total_mb\x18\a \x01(\x03R\rmemoryTotalMb\x12'\n" +
|
||||
"\x0fengines_changed\x18\b \x01(\bR\x0eenginesChanged\x12P\n" +
|
||||
"\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x1aA\n" +
|
||||
"\x13EngineVersionsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbb\x03\n" +
|
||||
"\vShardStatus\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" +
|
||||
"\n" +
|
||||
@@ -1998,7 +2035,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -2020,7 +2057,8 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*Ack)(nil), // 17: llmgw.v1.Ack
|
||||
nil, // 18: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 19: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 20: llmgw.v1.EngineProfile.EnvEntry
|
||||
nil, // 20: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 21: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -2041,17 +2079,18 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
0, // 18: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
11, // 19: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
13, // 20: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
20, // 21: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 22: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 23: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
23, // [23:24] is the sub-list for method output_type
|
||||
22, // [22:23] is the sub-list for method input_type
|
||||
22, // [22:22] is the sub-list for extension type_name
|
||||
22, // [22:22] is the sub-list for extension extendee
|
||||
0, // [0:22] is the sub-list for field type_name
|
||||
20, // 18: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
0, // 19: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
11, // 20: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
13, // 21: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
21, // 22: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 23: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 24: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
24, // [24:25] is the sub-list for method output_type
|
||||
23, // [23:24] is the sub-list for method input_type
|
||||
23, // [23:23] is the sub-list for extension type_name
|
||||
23, // [23:23] is the sub-list for extension extendee
|
||||
0, // [0:23] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -2077,7 +2116,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 20,
|
||||
NumMessages: 21,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -222,6 +222,18 @@ message Heartbeat {
|
||||
// report", and the first would quietly empty a node's inventory.
|
||||
bool weights_changed = 5;
|
||||
repeated LocalWeights weights = 6;
|
||||
|
||||
// memory_total_mb for the same reason again: RAM is added to a virtual
|
||||
// machine while it runs, or across a reboot the agent does not notice, and a
|
||||
// figure read once at registration kept a node that had grown to 32 GB
|
||||
// looking like the 7 GB machine it used to be. 0 means not read.
|
||||
int64 memory_total_mb = 7;
|
||||
|
||||
// engine_versions only when they changed, like weights. An engine installed
|
||||
// after the agent started is otherwise invisible to placement until
|
||||
// somebody restarts the agent — which is a step nobody knows to take.
|
||||
bool engines_changed = 8;
|
||||
map<string, string> engine_versions = 9;
|
||||
}
|
||||
|
||||
// ShardStatus reports a shard's lifecycle transition.
|
||||
|
||||
@@ -29,9 +29,11 @@ usage: $0 --node ID --addr HOST --server URL [options]
|
||||
--fabric NAME:hca1,hca2 . Omit for a node with no fabric,
|
||||
which can host single-node replicas and will never be given
|
||||
a multi-node one.
|
||||
--mgmt-addr ADDR address engines are reachable on (default: --addr)
|
||||
--mgmt-if NAME interface NCCL bootstraps over (default eth0)
|
||||
--llamacpp-bin P path to llama-server
|
||||
--mgmt-addr ADDR address engines are reachable on (default: the node's
|
||||
address on the route to --server, found by the agent)
|
||||
--mgmt-if NAME interface NCCL bootstraps over (default: the one holding
|
||||
that address)
|
||||
--llamacpp-bin P path to llama-server (default: PATH, then /opt/llama.cpp)
|
||||
--vllm-bin P path to the vllm entry point
|
||||
--sglang-python P python interpreter of the SGLang virtualenv
|
||||
--nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
|
||||
@@ -47,7 +49,7 @@ USAGE
|
||||
exit 2
|
||||
}
|
||||
|
||||
NODE= ADDR= SERVER= MGMT= MGMT_IF=eth0
|
||||
NODE= ADDR= SERVER= MGMT_OVERRIDE= MGMT_IF_OVERRIDE=
|
||||
NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
|
||||
LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
|
||||
SSH_WAIT=180
|
||||
@@ -60,8 +62,8 @@ while [ $# -gt 0 ]; do
|
||||
--addr) ADDR="$2"; shift 2;;
|
||||
--server) SERVER="$2"; shift 2;;
|
||||
--fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;;
|
||||
--mgmt-addr) MGMT="$2"; shift 2;;
|
||||
--mgmt-if) MGMT_IF="$2"; shift 2;;
|
||||
--mgmt-addr) MGMT_OVERRIDE="$2"; shift 2;;
|
||||
--mgmt-if) MGMT_IF_OVERRIDE="$2"; shift 2;;
|
||||
--llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;;
|
||||
--vllm-bin) VLLM_BIN="$2"; shift 2;;
|
||||
--sglang-python) SGLANG_PYTHON="$2"; shift 2;;
|
||||
@@ -74,7 +76,6 @@ while [ $# -gt 0 ]; do
|
||||
esac
|
||||
done
|
||||
[ -n "$NODE" ] && [ -n "$ADDR" ] && [ -n "$SERVER" ] || usage
|
||||
MGMT="${MGMT:-$ADDR}"
|
||||
BINARY="${BINARY:-$(command -v llmgw)}"
|
||||
[ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; }
|
||||
[ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; }
|
||||
@@ -135,63 +136,49 @@ scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
|
||||
$SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
|
||||
|
||||
say "writing the configuration"
|
||||
FABRICS="fabrics: []"
|
||||
if [ -n "$FABRIC_NAME" ]; then
|
||||
FABRICS="fabrics:
|
||||
- name: $FABRIC_NAME
|
||||
hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]"
|
||||
fi
|
||||
cat > "$TMP/agent.yaml" <<YAML
|
||||
# $NODE
|
||||
#
|
||||
# Written by onboard-node.sh. Fabric membership is configuration and there is no
|
||||
# way around it: OpenSM's default subnet prefix is the same on every fabric, so
|
||||
# a matching prefix proves nothing and a wrong guess produces a replica that
|
||||
# starts, passes health checks and then hangs on its first all-reduce.
|
||||
node_id: $NODE
|
||||
server: "$SERVER"
|
||||
|
||||
tls:
|
||||
ca: /etc/llmgw/ca.pem
|
||||
cert: /etc/llmgw/agent.pem
|
||||
key: /etc/llmgw/agent-key.pem
|
||||
|
||||
mgmt_addr: "$MGMT"
|
||||
mgmt_interface: $MGMT_IF
|
||||
|
||||
heartbeat_interval: 5s
|
||||
reconnect_base: 1s
|
||||
reconnect_max: 30s
|
||||
|
||||
$FABRICS
|
||||
|
||||
discovery:
|
||||
# Empty means look on PATH. Worth setting where the driver lives outside the
|
||||
# default paths, which containerized installs routinely do.
|
||||
nvidia_smi: "$NVIDIA_SMI"
|
||||
ib_root: $IB_ROOT
|
||||
prefer_nvml: false
|
||||
skip_gpus: false
|
||||
|
||||
# Where the engines live. Empty means "on PATH", which is rarely true: a node
|
||||
# that cannot find its engine registers healthy and then fails every shard
|
||||
# placed on it with "executable file not found".
|
||||
engines:
|
||||
llamacpp:
|
||||
bin: "$LLAMACPP_BIN"
|
||||
parallel: 4
|
||||
vllm:
|
||||
bin: "$VLLM_BIN"
|
||||
launch_mode: ray
|
||||
sglang:
|
||||
python: "$SGLANG_PYTHON"
|
||||
|
||||
supervisor:
|
||||
work_dir: /var/lib/llmgw-agent
|
||||
log_lines: 512
|
||||
stop_timeout: 60s
|
||||
health_interval: 2s
|
||||
YAML
|
||||
NODE_IS_HOSTNAME=no
|
||||
[ "$($SSH hostname 2>/dev/null)" = "$NODE" ] && NODE_IS_HOSTNAME=yes
|
||||
# Only what the node cannot find out for itself. Its name is its hostname, its
|
||||
# address is the one on the route to the control plane, llama-server is looked
|
||||
# for on PATH and where release archives are unpacked, GPUs are there when
|
||||
# nvidia-smi is, and memory, disk and engines are re-read while it runs. Every
|
||||
# setting written here is one that has to be kept true by hand — so an option
|
||||
# below is written only when it was given, as an override.
|
||||
{
|
||||
echo "# $NODE, written by onboard-node.sh."
|
||||
echo "server: \"$SERVER\""
|
||||
echo
|
||||
echo "tls:"
|
||||
echo " ca: /etc/llmgw/ca.pem"
|
||||
echo " cert: /etc/llmgw/agent.pem"
|
||||
echo " key: /etc/llmgw/agent-key.pem"
|
||||
# The certificate's common name is --node, and the control plane refuses a
|
||||
# stream whose node id differs from it. Said explicitly only when the
|
||||
# hostname would say something else.
|
||||
[ "$NODE_IS_HOSTNAME" = yes ] || echo "node_id: $NODE"
|
||||
[ -n "$MGMT_OVERRIDE" ] && echo "mgmt_addr: \"$MGMT_OVERRIDE\""
|
||||
[ -n "$MGMT_IF_OVERRIDE" ] && echo "mgmt_interface: $MGMT_IF_OVERRIDE"
|
||||
if [ -n "$FABRIC_NAME" ]; then
|
||||
# Fabric membership is the one thing that cannot be found: OpenSM's default
|
||||
# subnet prefix is the same on every fabric, so a matching prefix proves
|
||||
# nothing and a wrong guess hangs the first all-reduce.
|
||||
echo "fabrics:"
|
||||
echo " - name: $FABRIC_NAME"
|
||||
echo " hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]"
|
||||
fi
|
||||
if [ -n "$NVIDIA_SMI" ] || [ "$IB_ROOT" != /sys/class/infiniband ]; then
|
||||
echo "discovery:"
|
||||
[ -n "$NVIDIA_SMI" ] && echo " nvidia_smi: \"$NVIDIA_SMI\""
|
||||
[ "$IB_ROOT" != /sys/class/infiniband ] && echo " ib_root: $IB_ROOT"
|
||||
fi
|
||||
if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
||||
echo "engines:"
|
||||
[ -n "$LLAMACPP_BIN" ] && printf ' llamacpp:\n bin: "%s"\n' "$LLAMACPP_BIN"
|
||||
[ -n "$VLLM_BIN" ] && printf ' vllm:\n bin: "%s"\n' "$VLLM_BIN"
|
||||
[ -n "$SGLANG_PYTHON" ] && printf ' sglang:\n python: "%s"\n' "$SGLANG_PYTHON"
|
||||
fi
|
||||
true
|
||||
} > "$TMP/agent.yaml"
|
||||
scp -q "$TMP/agent.yaml" "$SSH_USER@$ADDR:/tmp/agent.yaml"
|
||||
|
||||
cat > "$TMP/llmgw-agent.service" <<'UNIT'
|
||||
|
||||
+22
-12
@@ -4,12 +4,19 @@
|
||||
# keep serving if the agent restarts: the router talks to them directly, and
|
||||
# orphan adoption picks them up again on the next start.
|
||||
|
||||
node_id: gpu-a-01 # defaults to the hostname
|
||||
# Everything a node can find out about itself, it does: its id is its hostname,
|
||||
# its address is the one on the route to the control plane, llama-server is
|
||||
# looked for on PATH and in /opt/llama.cpp, GPUs are there when nvidia-smi is,
|
||||
# weights are fetched into /var/lib/llmgw-weights and looked for there, and
|
||||
# memory, disk and installed engines are re-read while the agent runs. So a
|
||||
# plain CPU node needs only the two blocks below, and the same file on every
|
||||
# node. What follows them are overrides, for when the defaults are wrong.
|
||||
|
||||
server: "http://llmgw-server.internal:7443"
|
||||
|
||||
# The agent's half of mutual TLS. The certificate's common name must be this
|
||||
# node's id, because the control plane compares the two and refuses the stream
|
||||
# when they disagree:
|
||||
# node's id — its hostname, unless node_id says otherwise — because the control
|
||||
# plane compares the two and refuses the stream when they disagree:
|
||||
#
|
||||
# llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki
|
||||
#
|
||||
@@ -22,13 +29,19 @@ server: "http://llmgw-server.internal:7443"
|
||||
# cert: /etc/llmgw/agent.pem
|
||||
# key: /etc/llmgw/agent-key.pem
|
||||
|
||||
# ---- overrides ---------------------------------------------------------------
|
||||
|
||||
# node_id: gpu-a-01 # when it is not the hostname
|
||||
|
||||
# The address engines are reachable on, and the address NCCL bootstraps over.
|
||||
# Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA.
|
||||
mgmt_addr: "10.0.0.11"
|
||||
mgmt_interface: eth0
|
||||
# mgmt_addr: "10.0.0.11"
|
||||
# mgmt_interface: eth0
|
||||
|
||||
labels:
|
||||
rack: r12
|
||||
# Free-form, for a profile's node_selector. Nothing needs them; do not restate
|
||||
# what is measured (memory, GPUs, engines), because a label goes stale.
|
||||
# labels:
|
||||
# rack: r12
|
||||
|
||||
heartbeat_interval: 5s
|
||||
reconnect_base: 1s
|
||||
@@ -55,12 +68,9 @@ discovery:
|
||||
# tensor-parallel group off the PCIe path inside a node. It needs a binary
|
||||
# built with `-tags nvml`; otherwise this silently falls back to nvidia-smi.
|
||||
prefer_nvml: true
|
||||
skip_gpus: false
|
||||
|
||||
# Where this node keeps model weights.
|
||||
#
|
||||
# Unset by default, and unset means the node is not asked to look — which is
|
||||
# not the same as having nothing, and constrains nothing. Storage differs at
|
||||
# Where this node keeps model weights besides /var/lib/llmgw-weights, which is
|
||||
# where fetched weights land and is always looked in. Storage differs at
|
||||
# every installation and llmgw deliberately does not own it: a Lustre mount, an
|
||||
# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy
|
||||
# made by hand all look identical from here.
|
||||
|
||||
+23
-7
@@ -330,6 +330,9 @@ into an outage is worse.
|
||||
|
||||
`string`
|
||||
|
||||
NodeID defaults to the hostname. It must equal the common name of the
|
||||
agent's certificate, so set it only where the two differ.
|
||||
|
||||
### `server`
|
||||
|
||||
`string`
|
||||
@@ -352,19 +355,30 @@ edit to every node's configuration.
|
||||
|
||||
`string`
|
||||
|
||||
Hostname defaults to the machine's.
|
||||
|
||||
### `mgmt_addr`
|
||||
|
||||
`string`
|
||||
|
||||
MgmtAddr is the address engines are reached on. It defaults to this
|
||||
machine's address on the route to the control plane, which is the one
|
||||
the control plane and the router can reach.
|
||||
|
||||
### `labels`
|
||||
|
||||
`map[string]string`
|
||||
|
||||
Labels are free-form facts about a node for a profile's node_selector.
|
||||
Nothing needs them: memory, GPUs, engines and disk are measured, and a
|
||||
label restating one of those goes stale the day the machine changes.
|
||||
|
||||
### `mgmt_interface`
|
||||
|
||||
`string`
|
||||
|
||||
MgmtInterface is the ethernet interface NCCL bootstraps over.
|
||||
MgmtInterface is the ethernet interface NCCL bootstraps over. It
|
||||
defaults to the interface holding MgmtAddr, then eth0.
|
||||
|
||||
### `fabrics`
|
||||
|
||||
@@ -411,10 +425,8 @@ Takes a [Config](#config) block.
|
||||
|
||||
`[]string`
|
||||
|
||||
ModelDirs are the directories this node keeps model weights in.
|
||||
|
||||
Empty by default, and empty means the node is not asked to look — which
|
||||
is not the same as having nothing. Storage differs at every installation,
|
||||
ModelDirs are the directories this node keeps model weights in, beyond
|
||||
WeightsCache, which is always looked in. Storage differs at every installation,
|
||||
so llmgw observes it rather than owning it: a Lustre mount, an image
|
||||
exported over the fabric, local NVMe and a copy made by hand all look
|
||||
identical from here, and placement only needs to know whether the file
|
||||
@@ -426,7 +438,8 @@ is present.
|
||||
|
||||
WeightsCache is where a remote source is downloaded to. Defaults to the
|
||||
first entry of ModelDirs, because a node that says where its models live
|
||||
has already answered this question.
|
||||
has already answered this question, and otherwise to
|
||||
/var/lib/llmgw-weights.
|
||||
|
||||
Fetching is the simple arrangement: every node keeps its own copy. The
|
||||
exotic ones need no code — weights on a shared filer or an image exported
|
||||
@@ -576,6 +589,8 @@ know that this page cannot guess.
|
||||
|
||||
Bin is the llama-server binary. It is the server, not the CLI: the
|
||||
gateway talks OpenAI over HTTP and needs the one that serves it.
|
||||
Empty looks on PATH, then where release archives are unpacked
|
||||
(/opt/llama.cpp).
|
||||
|
||||
### `parallel`
|
||||
|
||||
@@ -761,7 +776,8 @@ source of NVLink topology.
|
||||
|
||||
`bool`
|
||||
|
||||
SkipGPUs omits GPU discovery, for nodes that have none.
|
||||
SkipGPUs omits GPU discovery. A node without nvidia-smi is already
|
||||
found to have none, so this is for one whose driver is broken.
|
||||
|
||||
## Prefix
|
||||
|
||||
|
||||
+192
-21
@@ -10,9 +10,11 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -35,6 +37,8 @@ import (
|
||||
|
||||
// Config configures the agent.
|
||||
type Config struct {
|
||||
// NodeID defaults to the hostname. It must equal the common name of the
|
||||
// agent's certificate, so set it only where the two differ.
|
||||
NodeID string `koanf:"node_id"`
|
||||
// Server is the control plane, and Servers is the same thing when there is
|
||||
// more than one. Both are accepted: every configuration written before
|
||||
@@ -45,13 +49,21 @@ type Config struct {
|
||||
// leader accepts an agent, so a standby is skipped in the time it takes to
|
||||
// be told so — which is what makes failover a reconnect rather than an
|
||||
// edit to every node's configuration.
|
||||
Server string `koanf:"server"`
|
||||
Servers []string `koanf:"servers"`
|
||||
Hostname string `koanf:"hostname"`
|
||||
MgmtAddr string `koanf:"mgmt_addr"`
|
||||
Labels map[string]string `koanf:"labels"`
|
||||
Server string `koanf:"server"`
|
||||
Servers []string `koanf:"servers"`
|
||||
// Hostname defaults to the machine's.
|
||||
Hostname string `koanf:"hostname"`
|
||||
// MgmtAddr is the address engines are reached on. It defaults to this
|
||||
// machine's address on the route to the control plane, which is the one
|
||||
// the control plane and the router can reach.
|
||||
MgmtAddr string `koanf:"mgmt_addr"`
|
||||
// Labels are free-form facts about a node for a profile's node_selector.
|
||||
// Nothing needs them: memory, GPUs, engines and disk are measured, and a
|
||||
// label restating one of those goes stale the day the machine changes.
|
||||
Labels map[string]string `koanf:"labels"`
|
||||
|
||||
// MgmtInterface is the ethernet interface NCCL bootstraps over.
|
||||
// MgmtInterface is the ethernet interface NCCL bootstraps over. It
|
||||
// defaults to the interface holding MgmtAddr, then eth0.
|
||||
MgmtInterface string `koanf:"mgmt_interface"`
|
||||
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
|
||||
// default subnet prefix is the same on every fabric, so nothing can be
|
||||
@@ -74,10 +86,8 @@ type Config struct {
|
||||
// keeps the plaintext behaviour, which only belongs on a trusted network.
|
||||
TLS tlsconf.Config `koanf:"tls"`
|
||||
|
||||
// ModelDirs are the directories this node keeps model weights in.
|
||||
//
|
||||
// Empty by default, and empty means the node is not asked to look — which
|
||||
// is not the same as having nothing. Storage differs at every installation,
|
||||
// ModelDirs are the directories this node keeps model weights in, beyond
|
||||
// WeightsCache, which is always looked in. Storage differs at every installation,
|
||||
// so llmgw observes it rather than owning it: a Lustre mount, an image
|
||||
// exported over the fabric, local NVMe and a copy made by hand all look
|
||||
// identical from here, and placement only needs to know whether the file
|
||||
@@ -86,7 +96,8 @@ type Config struct {
|
||||
|
||||
// WeightsCache is where a remote source is downloaded to. Defaults to the
|
||||
// first entry of ModelDirs, because a node that says where its models live
|
||||
// has already answered this question.
|
||||
// has already answered this question, and otherwise to
|
||||
// /var/lib/llmgw-weights.
|
||||
//
|
||||
// Fetching is the simple arrangement: every node keeps its own copy. The
|
||||
// exotic ones need no code — weights on a shared filer or an image exported
|
||||
@@ -116,18 +127,47 @@ func DefaultConfig() Config {
|
||||
|
||||
func (c *Config) normalize() error {
|
||||
d := DefaultConfig()
|
||||
if c.NodeID == "" {
|
||||
return errors.New("node_id is required")
|
||||
}
|
||||
if err := c.TLS.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.normalizeServerURL(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Everything below is found rather than written. A file that has to say
|
||||
// which machine it is on, and at what address, is a file that differs on
|
||||
// every node — and a copy taken from one node to the next registers the
|
||||
// second as the first.
|
||||
host, _ := os.Hostname()
|
||||
if c.NodeID == "" {
|
||||
c.NodeID = host
|
||||
}
|
||||
if c.NodeID == "" {
|
||||
return errors.New("node_id is not set and the hostname could not be read")
|
||||
}
|
||||
if c.Hostname == "" {
|
||||
c.Hostname = host
|
||||
}
|
||||
if c.MgmtAddr == "" {
|
||||
c.MgmtAddr = addrToward(c.Server)
|
||||
}
|
||||
if c.MgmtInterface == "" {
|
||||
c.MgmtInterface = interfaceOf(c.MgmtAddr)
|
||||
}
|
||||
if c.MgmtInterface == "" {
|
||||
c.MgmtInterface = d.MgmtInterface
|
||||
}
|
||||
// The directory downloads land in is always one that is looked in. When
|
||||
// they were two settings, a node could fill its disk with fetched weights
|
||||
// that placement never saw — which is what every node here was doing.
|
||||
if c.WeightsCache == "" && len(c.ModelDirs) > 0 {
|
||||
c.WeightsCache = c.ModelDirs[0]
|
||||
}
|
||||
if c.WeightsCache == "" {
|
||||
c.WeightsCache = DefaultWeightsCache
|
||||
}
|
||||
if !slices.Contains(c.ModelDirs, c.WeightsCache) {
|
||||
c.ModelDirs = append(c.ModelDirs, c.WeightsCache)
|
||||
}
|
||||
if c.HeartbeatInterval <= 0 {
|
||||
c.HeartbeatInterval = d.HeartbeatInterval
|
||||
}
|
||||
@@ -180,6 +220,61 @@ func (c *Config) normalizeServerURL() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultWeightsCache is where fetched weights go when nothing says otherwise.
|
||||
const DefaultWeightsCache = "/var/lib/llmgw-weights"
|
||||
|
||||
// addrToward is this machine's address on the route to the control plane.
|
||||
//
|
||||
// The address engines are reached on is whichever one the control plane can
|
||||
// reach, and the kernel already knows which that is: a UDP "connection" picks
|
||||
// the source address for the route without sending a packet. Asking the
|
||||
// interfaces instead would have to guess among docker bridges, VPNs and
|
||||
// link-local addresses.
|
||||
func addrToward(server string) string {
|
||||
u, err := url.Parse(server)
|
||||
if err != nil || u.Host == "" {
|
||||
return ""
|
||||
}
|
||||
host := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "443"
|
||||
}
|
||||
conn, err := net.Dial("udp", net.JoinHostPort(host, port))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer conn.Close()
|
||||
if a, ok := conn.LocalAddr().(*net.UDPAddr); ok {
|
||||
return a.IP.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// interfaceOf names the interface that holds an address, or "" when none does.
|
||||
func interfaceOf(addr string) string {
|
||||
ip := net.ParseIP(addr)
|
||||
if ip == nil {
|
||||
return ""
|
||||
}
|
||||
ifaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, ifc := range ifaces {
|
||||
addrs, err := ifc.Addrs()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range addrs {
|
||||
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
|
||||
return ifc.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// normalizeOne applies the scheme rules to a single address.
|
||||
func normalizeOne(addr string, secure bool) (string, error) {
|
||||
switch {
|
||||
@@ -209,13 +304,22 @@ type Agent struct {
|
||||
reg *engines.Registry
|
||||
http connectHTTPClient
|
||||
|
||||
// engineOnce probes the installed engines exactly once.
|
||||
// engineOnce probes the installed engines once before registering.
|
||||
//
|
||||
// Once, because probing runs each engine's --version and vLLM's imports
|
||||
// torch — seconds on a cold cache. A reconnect is not a reason to pay that
|
||||
// again, and what is installed does not change while the process runs.
|
||||
// Once, because probing SGLang imports it in a Python interpreter — seconds
|
||||
// on a cold cache — and a reconnect is not a reason to pay that again.
|
||||
// Later changes are found by the slower re-probe the heartbeat starts, not
|
||||
// by registering again: an engine installed while the agent runs used to
|
||||
// stay invisible until somebody thought to restart the agent.
|
||||
engineOnce sync.Once
|
||||
// engineVers is guarded by mu once the first probe has run.
|
||||
engineVers map[string]string
|
||||
// enginesSent is the fingerprint of the engines the control plane last
|
||||
// heard about, from registration or a heartbeat. Guarded by mu.
|
||||
enginesSent string
|
||||
// lastEngineProbe and probingEngines pace the re-probe. Guarded by mu.
|
||||
lastEngineProbe time.Time
|
||||
probingEngines bool
|
||||
|
||||
mu sync.Mutex
|
||||
inv discovery.Inventory
|
||||
@@ -668,6 +772,21 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
a.mu.Unlock()
|
||||
|
||||
hb := &llmgwv1.Heartbeat{NodeId: a.cfg.NodeID}
|
||||
|
||||
// Read every time; it is one small file. Memory is added to a running VM,
|
||||
// and a node that grew from 7 GB to 32 GB has to be seen to have grown
|
||||
// without anybody restarting anything.
|
||||
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
|
||||
|
||||
a.reprobeEngines()
|
||||
a.mu.Lock()
|
||||
if fp := enginesFingerprint(a.engineVers); a.engineVers != nil && fp != a.enginesSent {
|
||||
a.enginesSent = fp
|
||||
hb.EnginesChanged = true
|
||||
hb.EngineVersions = maps.Clone(a.engineVers)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
for _, g := range inv.GPUs {
|
||||
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
|
||||
}
|
||||
@@ -1041,9 +1160,61 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
|
||||
// never going to be asked to run.
|
||||
func (a *Agent) engines(ctx context.Context) map[string]string {
|
||||
a.engineOnce.Do(func() {
|
||||
a.engineVers = a.reg.Probe(ctx)
|
||||
vers := a.reg.Probe(ctx)
|
||||
a.mu.Lock()
|
||||
a.engineVers, a.lastEngineProbe = vers, time.Now()
|
||||
a.mu.Unlock()
|
||||
a.log.Info("engines available on this node",
|
||||
"engines", engines.ProbedNames(a.engineVers))
|
||||
"engines", engines.ProbedNames(vers))
|
||||
})
|
||||
return a.engineVers
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
// Whatever registration sends is what the control plane now knows, so a
|
||||
// heartbeat repeats it only once it differs.
|
||||
a.enginesSent = enginesFingerprint(a.engineVers)
|
||||
return maps.Clone(a.engineVers)
|
||||
}
|
||||
|
||||
// engineReprobe is how often the installed engines are looked for again.
|
||||
//
|
||||
// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
|
||||
// minute's delay in noticing is invisible, while SGLang's probe starts a
|
||||
// Python interpreter that imports it.
|
||||
const engineReprobe = time.Minute
|
||||
|
||||
// reprobeEngines looks for installed engines again when it is time, off the
|
||||
// heartbeat goroutine so a slow probe never delays a heartbeat.
|
||||
func (a *Agent) reprobeEngines() {
|
||||
a.mu.Lock()
|
||||
due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
|
||||
time.Since(a.lastEngineProbe) >= engineReprobe
|
||||
if due {
|
||||
a.probingEngines = true
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if !due {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
vers := a.reg.Probe(ctx)
|
||||
|
||||
a.mu.Lock()
|
||||
changed := enginesFingerprint(vers) != enginesFingerprint(a.engineVers)
|
||||
a.engineVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
|
||||
a.mu.Unlock()
|
||||
if changed {
|
||||
a.log.Info("engines available on this node changed",
|
||||
"engines", engines.ProbedNames(vers))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func enginesFingerprint(vers map[string]string) string {
|
||||
var b strings.Builder
|
||||
for _, k := range slices.Sorted(maps.Keys(vers)) {
|
||||
fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -24,7 +24,8 @@ type Options struct {
|
||||
// PreferNVML uses the NVML backend when this binary has one. It is the only
|
||||
// source of NVLink topology.
|
||||
PreferNVML bool `koanf:"prefer_nvml"`
|
||||
// SkipGPUs omits GPU discovery, for nodes that have none.
|
||||
// SkipGPUs omits GPU discovery. A node without nvidia-smi is already
|
||||
// found to have none, so this is for one whose driver is broken.
|
||||
SkipGPUs bool `koanf:"skip_gpus"`
|
||||
// ModelDirs are scanned for weights. Empty means do not look.
|
||||
ModelDirs []string `koanf:"-"`
|
||||
|
||||
@@ -44,6 +44,13 @@ const mandatorySMIFields = 10
|
||||
// NVML build adds.
|
||||
func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
|
||||
if binary == "" {
|
||||
// Not configured and not installed is a node without an NVIDIA card,
|
||||
// not a failed discovery. Reported as an error it also cut discovery
|
||||
// short before InfiniBand was looked at, on every CPU node that had
|
||||
// not been told by hand to skip GPUs.
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
binary = "nvidia-smi"
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, binary,
|
||||
|
||||
@@ -3,6 +3,7 @@ package engines
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -13,6 +14,8 @@ import (
|
||||
type LlamaCppConfig struct {
|
||||
// Bin is the llama-server binary. It is the server, not the CLI: the
|
||||
// gateway talks OpenAI over HTTP and needs the one that serves it.
|
||||
// Empty looks on PATH, then where release archives are unpacked
|
||||
// (/opt/llama.cpp).
|
||||
Bin string `koanf:"bin"`
|
||||
// Parallel is how many requests a server handles at once, when a profile
|
||||
// does not say. llama.cpp defaults to one, which would make every second
|
||||
@@ -68,10 +71,7 @@ func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
|
||||
|
||||
// Commands returns the processes for one shard, which is always one process.
|
||||
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
bin := l.cfg.Bin
|
||||
if bin == "" {
|
||||
bin = "llama-server"
|
||||
}
|
||||
bin := l.bin()
|
||||
p := s.Profile
|
||||
|
||||
model, err := ggufIn(p.ModelPath)
|
||||
@@ -169,3 +169,34 @@ func ggufIn(path string) (string, error) {
|
||||
sort.Strings(names)
|
||||
return filepath.Join(path, names[0]), nil
|
||||
}
|
||||
|
||||
// llamaServerPlaces are where llama.cpp's release archives are unpacked when
|
||||
// they are not on PATH. The archive carries its shared libraries beside the
|
||||
// binary, so it is usually left in one directory rather than installed.
|
||||
var llamaServerPlaces = []string{
|
||||
"/opt/llama.cpp/llama-server",
|
||||
"/opt/llama.cpp/bin/llama-server",
|
||||
"/opt/llama.cpp/build/bin/llama-server",
|
||||
"/usr/local/llama.cpp/llama-server",
|
||||
}
|
||||
|
||||
// bin is the llama-server to run: the configured one, else the one on PATH,
|
||||
// else the first of the usual unpack locations that exists.
|
||||
//
|
||||
// Found rather than required in the configuration, because every node here
|
||||
// had the same path written into its file by hand, and a node set up without
|
||||
// it reported no engine at all.
|
||||
func (l *LlamaCpp) bin() string {
|
||||
if l.cfg.Bin != "" {
|
||||
return l.cfg.Bin
|
||||
}
|
||||
if p, err := exec.LookPath("llama-server"); err == nil {
|
||||
return p
|
||||
}
|
||||
for _, p := range llamaServerPlaces {
|
||||
if info, err := os.Stat(p); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return "llama-server"
|
||||
}
|
||||
@@ -159,11 +159,7 @@ func firstVersion(s string) string {
|
||||
|
||||
// Probe reports whether the llama.cpp server is installed.
|
||||
func (l *LlamaCpp) Probe(context.Context) (string, error) {
|
||||
bin := l.cfg.Bin
|
||||
if bin == "" {
|
||||
bin = "llama-server"
|
||||
}
|
||||
return installedBinary(bin)
|
||||
return installedBinary(l.bin())
|
||||
}
|
||||
|
||||
// Probe reports whether vLLM is installed.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
func TestAnAgentConfigNeedsOnlyWhereTheControlPlaneIs(t *testing.T) {
|
||||
// Every node carried its own name, its own address and the same two
|
||||
// directories written out by hand, so no two agent files were alike and a
|
||||
// copy taken from one node registered the next as the first.
|
||||
cfg := Config{Server: "127.0.0.1:7443"}
|
||||
if err := cfg.normalize(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
if cfg.NodeID != host || cfg.Hostname != host {
|
||||
t.Errorf("node_id %q, hostname %q; expected the machine's name %q",
|
||||
cfg.NodeID, cfg.Hostname, host)
|
||||
}
|
||||
if cfg.MgmtAddr != "127.0.0.1" {
|
||||
t.Errorf("mgmt_addr %q, expected the address on the route to the control plane",
|
||||
cfg.MgmtAddr)
|
||||
}
|
||||
if cfg.WeightsCache != DefaultWeightsCache {
|
||||
t.Errorf("weights land in %q", cfg.WeightsCache)
|
||||
}
|
||||
if !slices.Contains(cfg.ModelDirs, cfg.WeightsCache) {
|
||||
t.Errorf("the download directory %q is not looked in: %v",
|
||||
cfg.WeightsCache, cfg.ModelDirs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatIsWrittenStillWins(t *testing.T) {
|
||||
cfg := Config{
|
||||
Server: "127.0.0.1:7443", NodeID: "gpu-a-01", MgmtAddr: "10.9.9.9",
|
||||
ModelDirs: []string{"/lustre/models"}, WeightsCache: "/scratch/w",
|
||||
}
|
||||
if err := cfg.normalize(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.NodeID != "gpu-a-01" || cfg.MgmtAddr != "10.9.9.9" {
|
||||
t.Errorf("configured identity overridden: %q %q", cfg.NodeID, cfg.MgmtAddr)
|
||||
}
|
||||
if !slices.Equal(cfg.ModelDirs, []string{"/lustre/models", "/scratch/w"}) {
|
||||
t.Errorf("model dirs %v", cfg.ModelDirs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) {
|
||||
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir()}
|
||||
cfg.Superv.WorkDir = t.TempDir()
|
||||
a, err := New(cfg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engines := func(m *llmgwv1.AgentMessage) *llmgwv1.Heartbeat { return m.GetHeartbeat() }
|
||||
|
||||
a.mu.Lock()
|
||||
a.engineVers = map[string]string{"llamacpp": "unknown"}
|
||||
a.enginesSent = enginesFingerprint(a.engineVers)
|
||||
a.mu.Unlock()
|
||||
if hb := engines(a.heartbeat()); hb.GetEnginesChanged() {
|
||||
t.Errorf("unchanged engines were resent")
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
a.engineVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"}
|
||||
a.mu.Unlock()
|
||||
hb := engines(a.heartbeat())
|
||||
if !hb.GetEnginesChanged() || len(hb.GetEngineVersions()) != 2 {
|
||||
t.Errorf("an installed engine was not reported: %v", hb.GetEngineVersions())
|
||||
}
|
||||
if engines(a.heartbeat()).GetEnginesChanged() {
|
||||
t.Errorf("reported again on the next heartbeat")
|
||||
}
|
||||
}
|
||||
@@ -39,8 +39,10 @@ func TestTheDocumentedAgentExampleLoads(t *testing.T) {
|
||||
if f.RateGbps == 0 {
|
||||
t.Error("the fabric's rate did not bind")
|
||||
}
|
||||
if cfg.NodeID == "" || cfg.Server == "" {
|
||||
t.Errorf("node_id=%q server=%q", cfg.NodeID, cfg.Server)
|
||||
// node_id is left to the hostname, so the example need not say it; where
|
||||
// the control plane is, nothing can find out.
|
||||
if cfg.Server == "" {
|
||||
t.Errorf("server=%q", cfg.Server)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -11,7 +12,8 @@ import (
|
||||
func nodeWith(s *Service, n inventory.Node) { s.inv.Upsert(n) }
|
||||
|
||||
func newTestService() *Service {
|
||||
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{}}
|
||||
return &Service{inv: inventory.NewStore(), conns: map[string]*conn{},
|
||||
log: slog.New(slog.DiscardHandler)}
|
||||
}
|
||||
|
||||
func TestDiskIsRefreshedWhileTheAgentRuns(t *testing.T) {
|
||||
@@ -112,3 +114,44 @@ func TestAHeartbeatAboutNothingChangesNothing(t *testing.T) {
|
||||
t.Errorf("disk changed: %d", free)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeartbeatCarriesMemoryAddedToARunningNode(t *testing.T) {
|
||||
// Read once at registration, a node that was given 32 GB went on looking
|
||||
// like the 7 GB machine it had been, and every model that needed the
|
||||
// difference stayed unplaced until somebody restarted its agent.
|
||||
s := newTestService()
|
||||
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, MemoryTotalMB: 7 * 1024})
|
||||
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 32 * 1024})
|
||||
|
||||
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
|
||||
t.Errorf("memory is %d MB, expected the 32 GB the node now has", n.MemoryTotalMB)
|
||||
}
|
||||
|
||||
// Zero is an agent that could not read it, not a node that lost its RAM.
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1"})
|
||||
if n, _ := s.inv.Get("n1"); n.MemoryTotalMB != 32*1024 {
|
||||
t.Errorf("a heartbeat without memory erased it: %d", n.MemoryTotalMB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeartbeatCarriesAnEngineInstalledAfterRegistration(t *testing.T) {
|
||||
s := newTestService()
|
||||
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready,
|
||||
EngineVersions: map[string]string{"llamacpp": "unknown"}})
|
||||
|
||||
// Unchanged engines are not resent, and silence must not clear them.
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{NodeId: "n1", MemoryTotalMb: 1024})
|
||||
if n, _ := s.inv.Get("n1"); len(n.EngineVersions) != 1 {
|
||||
t.Fatalf("engines changed without being reported: %v", n.EngineVersions)
|
||||
}
|
||||
|
||||
s.applyHeartbeat("n1", &llmgwv1.Heartbeat{
|
||||
NodeId: "n1", EnginesChanged: true,
|
||||
EngineVersions: map[string]string{"llamacpp": "unknown", "vllm": "unknown"},
|
||||
})
|
||||
n, _ := s.inv.Get("n1")
|
||||
if _, ok := n.EngineVersions["vllm"]; !ok {
|
||||
t.Errorf("the newly installed engine is not known: %v", n.EngineVersions)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sync"
|
||||
@@ -269,7 +270,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
used := hb.GetGpuMemoryUsedMb()
|
||||
problems := hb.GetNewProblems()
|
||||
storage := hb.GetStorage()
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() {
|
||||
memMB := int(hb.GetMemoryTotalMb())
|
||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -316,6 +319,26 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// RAM and engines change under a running agent too: memory added to a VM,
|
||||
// an engine installed this afternoon. Said in the log when they do, because
|
||||
// "the node now qualifies" is the answer to why a model placed an hour after
|
||||
// it was asked for, and "the node no longer does" is the answer to why one
|
||||
// stopped placing.
|
||||
if memMB > 0 && memMB != n.MemoryTotalMB {
|
||||
s.log.Info("node memory changed", "node", nodeID,
|
||||
"from_mb", n.MemoryTotalMB, "to_mb", memMB)
|
||||
n.MemoryTotalMB = memMB
|
||||
}
|
||||
if hb.GetEnginesChanged() {
|
||||
vers := maps.Clone(hb.GetEngineVersions())
|
||||
if !maps.Equal(vers, n.EngineVersions) {
|
||||
s.log.Info("node engines changed", "node", nodeID,
|
||||
"from", slices.Sorted(maps.Keys(n.EngineVersions)),
|
||||
"to", slices.Sorted(maps.Keys(vers)))
|
||||
}
|
||||
n.EngineVersions = vers
|
||||
}
|
||||
s.inv.Upsert(n)
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user