llama.cpp's start-up output is read as a shard becomes ready: the devices it used, its backend, and the layers it offloaded. That goes up with READY in a new ShardStatus.compute, is kept on the shard, and is shown by the admin API, ctl replicas (ON) and the console (COMPUTE). A shard given GPUs whose engine says it found no usable GPU or offloaded no layer is stopped and failed, even where nvidia-smi cannot be asked. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
863 lines
33 KiB
Protocol Buffer
863 lines
33 KiB
Protocol Buffer
syntax = "proto3";
|
||
|
||
// Package llmgw.v1 is the protocol between a node agent and the control plane
|
||
// (plan §5.6).
|
||
//
|
||
// The agent dials out and holds a bidirectional stream open. That direction is
|
||
// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
|
||
// control plane does, and an outbound connection needs no inbound rule on every
|
||
// node in the fleet.
|
||
package llmgw.v1;
|
||
|
||
option go_package = "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1;llmgwv1";
|
||
|
||
service AgentService {
|
||
// Connect opens the agent's stream. It stays open for the agent's lifetime;
|
||
// losing it is how each side learns the other is gone.
|
||
rpc Connect(stream AgentMessage) returns (stream ControlMessage);
|
||
}
|
||
|
||
// AgentMessage is anything a node tells the control plane.
|
||
message AgentMessage {
|
||
oneof msg {
|
||
Register register = 1;
|
||
Heartbeat heartbeat = 2;
|
||
ShardStatus shard = 3;
|
||
LogsReply logs = 4;
|
||
FabricScanReply fabric_scan = 5;
|
||
FabricTestReply fabric_test = 6;
|
||
UpgradeReply upgrade = 7;
|
||
KVEvents kv_events = 8;
|
||
}
|
||
}
|
||
|
||
// KVEvents is a change to what one vLLM replica's prefix cache holds, read
|
||
// by the agent from the engine's KV cache events and sent on for the router's
|
||
// precise index (plan §7.5 tier 2).
|
||
//
|
||
// A net effect since the last one, not the events themselves: a block stored
|
||
// and evicted between two sends is not worth the bytes. On every new stream
|
||
// the agent first sends each replica in full, with clear_first set, because a
|
||
// control plane that has just started or taken over knows nothing.
|
||
message KVEvents {
|
||
string replica_id = 1;
|
||
// block_size is how many tokens make one of the replica's blocks.
|
||
int32 block_size = 2;
|
||
// clear_first says to forget what was known of the replica before applying the
|
||
// rest: its cache was cleared, its engine restarted, events were lost, or
|
||
// this is the first piece of a full copy.
|
||
bool clear_first = 3;
|
||
// stored and removed are block hashes as vLLM publishes them: the low 64
|
||
// bits of the sha256_cbor digest.
|
||
repeated fixed64 stored = 4;
|
||
repeated fixed64 removed = 5;
|
||
// problem, when set, is why the replica's events cannot be used — most
|
||
// likely an engine hashing another way than llmgw configured. Its index is
|
||
// empty while it stands.
|
||
string problem = 6;
|
||
}
|
||
|
||
// ControlMessage is anything the control plane tells a node.
|
||
message ControlMessage {
|
||
oneof msg {
|
||
StartShard start = 1;
|
||
StopShard stop = 2;
|
||
DrainShard drain = 3;
|
||
Ack ack = 4;
|
||
RemoveWeights remove_weights = 5;
|
||
Provision provision = 6;
|
||
LogsRequest logs = 7;
|
||
FabricScanRequest fabric_scan = 8;
|
||
FabricTestRequest fabric_test = 9;
|
||
UpgradeAgent upgrade = 10;
|
||
ControlPlanes control_planes = 11;
|
||
Firewall firewall = 13;
|
||
}
|
||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||
// is one) of the control plane's span that sent this command, so what the
|
||
// agent does for it — fetching weights, launching the engine, waiting for
|
||
// it to answer — lands in the same trace. Empty when the control plane is
|
||
// not tracing, and ignored by an agent that is not.
|
||
map<string, string> trace = 12;
|
||
}
|
||
|
||
// Firewall is whether llmgw's firewall is kept on the node — its nftables
|
||
// table, which lets the cluster's networks alone reach the engines' ports and
|
||
// the node exporter — sent when an agent registers and whenever the
|
||
// cluster's settings change it (firewall.enabled).
|
||
message Firewall {
|
||
bool enabled = 1;
|
||
// networks are let in besides those the node reaches the control planes
|
||
// on: firewall_networks, networks or single addresses.
|
||
repeated string networks = 2;
|
||
// engine_ports_from and engine_ports_to are the engines' ports
|
||
// (control.port_range).
|
||
int32 engine_ports_from = 3;
|
||
int32 engine_ports_to = 4;
|
||
}
|
||
|
||
// ControlPlanes is every control plane an agent may dial, sent by the leader
|
||
// when an agent registers and whenever the list changes. The agent keeps it
|
||
// beside its configuration, so a second control plane added after the node
|
||
// was onboarded is one it fails over to without anybody editing the node.
|
||
message ControlPlanes {
|
||
repeated string servers = 1;
|
||
}
|
||
|
||
// Register is the first message on every stream. It carries the node's full
|
||
// hardware inventory, so the control plane needs no separate discovery call and
|
||
// a reconnecting agent automatically refreshes what the scheduler knows.
|
||
message Register {
|
||
string node_id = 1;
|
||
string hostname = 2;
|
||
// mgmt_addr is the address engines are reachable on, and the address NCCL
|
||
// bootstraps over. Never a fabric address: NCCL rendezvouses over TCP before
|
||
// it moves to RDMA.
|
||
string mgmt_addr = 3;
|
||
map<string, string> labels = 4;
|
||
|
||
repeated GPU gpus = 5;
|
||
// fabrics is empty on a node without InfiniBand. Such a node is its own
|
||
// single-node island and can never host a multi-node replica.
|
||
repeated FabricAttachment fabrics = 6;
|
||
|
||
string agent_version = 7;
|
||
// engine_versions decides which CLI flags and metric names apply; both drift
|
||
// between engine releases (plan §14).
|
||
map<string, string> engine_versions = 8;
|
||
|
||
// running_shards are the engines this agent is already supervising, including
|
||
// ones adopted from a previous agent.
|
||
//
|
||
// This is what lets a control plane restart recover instead of orphaning live
|
||
// engines. Without it the control plane forgets every replica, places new
|
||
// ones, and the old engines keep holding GPUs the inventory believes are free
|
||
// — so the new engines fail to allocate, with an error naming the wrong
|
||
// process (plan §6.5).
|
||
repeated RunningShard running_shards = 9;
|
||
|
||
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
||
// which is what every agent older than this field reports, and is treated as
|
||
// no constraint rather than as a node with no memory.
|
||
//
|
||
// Placement needs it because a CPU profile has no card to be free, so every
|
||
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
||
// would be placed just as happily on a node with 8 GB.
|
||
int64 memory_total_mb = 10;
|
||
|
||
// weights are the model files this node can see, from the directories its
|
||
// configuration names.
|
||
//
|
||
// How they got there is the operator's business and deliberately outside
|
||
// llmgw's: a shared filesystem, an image exported over the fabric, rsync, or
|
||
// a copy made by hand all look identical here. What matters to placement is
|
||
// only whether they are present, because a node without them either cannot
|
||
// start the replica at all or must first move tens of gigabytes.
|
||
//
|
||
// Empty means the node was not asked to look, which is not the same as having
|
||
// nothing — an agent with no model directories configured behaves exactly as
|
||
// every agent did before this existed, and constrains nothing.
|
||
repeated LocalWeights weights = 11;
|
||
|
||
// storage is the disk behind those weights, one entry per filesystem.
|
||
//
|
||
// Reported so an operator can see a node filling up before a fetch dies at
|
||
// ninety percent, and so placement can refuse a node that cannot hold a
|
||
// model rather than discovering it during the download.
|
||
//
|
||
// Empty means the node was not asked to look, exactly as for weights, and
|
||
// constrains nothing.
|
||
repeated Storage storage = 12;
|
||
|
||
// host is what the machine is, so the control plane can have the mirror
|
||
// fetch software built for it rather than assuming every node matches the
|
||
// first.
|
||
Host host = 13;
|
||
|
||
// engines_probed says engine_versions is an answer. Without it an empty map
|
||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||
// built node with no engine installed yet was a candidate for every model.
|
||
bool engines_probed = 14;
|
||
// cpu_count is the CPUs the agent may run on. Placement reserves cores
|
||
// for CPU replicas out of it, the way it reserves memory out of
|
||
// memory_total_mb; zero, from an older agent, reserves nothing.
|
||
int32 cpu_count = 18;
|
||
// imex is the service that lets a multi-node NVLink domain share memory.
|
||
IMEX imex = 15;
|
||
|
||
// installable_engines are the engines this agent knows how to install
|
||
// itself. The control plane asks it for no others. Empty is an agent older
|
||
// than the field, which could install llama.cpp and nothing else: asked for
|
||
// vLLM, it answered "unsupported", and every node of a cluster half way
|
||
// through an upgrade showed as failing.
|
||
repeated string installable_engines = 16;
|
||
|
||
// features are the requests this agent answers beyond the ones every agent
|
||
// does: "fabric-test". An agent that does not know a request ignores it, so
|
||
// without this the control plane would wait minutes for an answer that is
|
||
// never coming.
|
||
repeated string features = 17;
|
||
}
|
||
|
||
// Host describes the operating system and processor of a node.
|
||
message Host {
|
||
// From /etc/os-release: "ubuntu", "24.04", "noble".
|
||
string os_id = 1;
|
||
string os_version = 2;
|
||
string os_codename = 3;
|
||
// Debian's spelling, which is what package indexes are keyed by: "amd64",
|
||
// "arm64".
|
||
string arch = 4;
|
||
// The vector extensions inference cares about, of those the CPU has. A VM
|
||
// given QEMU's default CPU model has none of them, and runs, and is five
|
||
// times slower, and nothing says why.
|
||
repeated string cpu_flags = 5;
|
||
// nvidia_driver is the driver version, empty when there is none. A card
|
||
// without a driver is not a GPU anything can use.
|
||
string nvidia_driver = 6;
|
||
}
|
||
|
||
// Storage is one filesystem a node keeps weights on.
|
||
//
|
||
// Per filesystem rather than per configured directory: free space belongs to
|
||
// the disk, and two model directories on one disk are one pool of bytes.
|
||
message Storage {
|
||
// path is one directory on this filesystem, so the operator recognises it.
|
||
string path = 1;
|
||
// free_bytes is what a non-root process can write — not the root reserve,
|
||
// which is space the agent cannot have.
|
||
int64 free_bytes = 2;
|
||
int64 total_bytes = 3;
|
||
// shared marks a filesystem the node did not fill. Free space on one is
|
||
// shared with every other node using it, so two nodes each reporting 500 GB
|
||
// free on the same filer have 500 GB between them and not a terabyte.
|
||
bool shared = 4;
|
||
}
|
||
|
||
// RunningShard describes an engine already running on a node.
|
||
//
|
||
// It carries enough for the control plane to rebuild the replica it belongs to,
|
||
// which is more than a shard id: the replica's identity, its pool and fabric,
|
||
// and the leader endpoint clients reach it through.
|
||
message RunningShard {
|
||
string replica_id = 1;
|
||
string shard_id = 2;
|
||
int32 rank = 3;
|
||
int32 nnodes = 4;
|
||
repeated int32 gpu_indices = 5;
|
||
repeated string hcas = 6;
|
||
int32 http_port = 7;
|
||
ShardStatus.State state = 8;
|
||
|
||
// model and profile name the catalog entry this shard was started from.
|
||
string model = 9;
|
||
string profile = 10;
|
||
// pool and fabric are echoed back from the StartShard that created it; the
|
||
// agent has no way to derive them.
|
||
string pool = 11;
|
||
string fabric = 12;
|
||
|
||
string leader_addr = 13;
|
||
int32 leader_port = 14;
|
||
|
||
// profile_fp is the fingerprint of the profile this shard was started from,
|
||
// echoed back from StartShard. The control plane keeps no record of a running
|
||
// replica across its own restart, so without this a shard adopted after a
|
||
// restart could not be compared with the catalog — which is exactly when the
|
||
// catalog is most likely to have been edited.
|
||
string profile_fp = 15;
|
||
}
|
||
|
||
message GPU {
|
||
int32 index = 1;
|
||
string uuid = 2;
|
||
string name = 3;
|
||
int32 memory_total_mb = 4;
|
||
int32 memory_used_mb = 5;
|
||
string compute_capability = 6;
|
||
int32 numa_node = 7;
|
||
string pci_bus_id = 8;
|
||
string driver_version = 9;
|
||
string cuda_version = 10;
|
||
// nvlink_peers is only populated by agents built with the nvml tag.
|
||
repeated int32 nvlink_peers = 11;
|
||
// problems keeps a GPU out of placement: ECC faults, or memory already held
|
||
// by another process.
|
||
repeated string problems = 12;
|
||
int32 temperature_c = 13;
|
||
int32 ecc_errors = 14;
|
||
// utilization_percent is how busy the card was over the last sample period.
|
||
// Zero when the driver is too old to report it, which is not the same as
|
||
// idle: the metric is simply absent rather than wrong.
|
||
int32 utilization_percent = 15;
|
||
// nic_paths is how far each network adapter is from this card, by HCA
|
||
// name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
|
||
map<string, string> nic_paths = 16;
|
||
// The card's multi-node NVLink membership, from nvidia-smi -q: state is
|
||
// Completed once the fabric manager has joined it; cluster_uuid and
|
||
// clique_id name the domain it joined.
|
||
string fabric_state = 17;
|
||
string fabric_status = 18;
|
||
string cluster_uuid = 19;
|
||
string clique_id = 20;
|
||
string fabric_bandwidth = 21;
|
||
// mig is set when this is a MIG instance rather than a whole card. index
|
||
// is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and
|
||
// memory the instance's share; utilization_percent stays zero, because no
|
||
// driver measures it per instance.
|
||
MIGDevice mig = 22;
|
||
}
|
||
|
||
// MIGDevice is which card a MIG instance is carved from, and how.
|
||
message MIGDevice {
|
||
int32 parent_index = 1;
|
||
string parent_uuid = 2;
|
||
// parent_memory_total_mb is the whole card's: the class is named from it.
|
||
int32 parent_memory_total_mb = 3;
|
||
// device is the instance's number on its card, as nvidia-smi -L lists it.
|
||
int32 device = 4;
|
||
// profile is NVIDIA's name for the slice, like 3g.40gb.
|
||
string profile = 5;
|
||
int32 gpu_instance = 6;
|
||
int32 compute_instance = 7;
|
||
}
|
||
|
||
message FabricAttachment {
|
||
// fabric is the operator-assigned name. It is not derived from the GID
|
||
// prefix: OpenSM defaults to fe80:: on every fabric, so a matching prefix
|
||
// proves nothing (plan §5.2).
|
||
string fabric = 1;
|
||
repeated string hcas = 2;
|
||
int32 rate_gbps = 3;
|
||
}
|
||
|
||
// Heartbeat says the node is alive and reports anything that changed cheaply.
|
||
// LocalWeights is one model file or directory a node can see.
|
||
message LocalWeights {
|
||
// path is what an engine would be given, so it can be compared against a
|
||
// model's source without interpretation.
|
||
string path = 1;
|
||
int64 size_bytes = 2;
|
||
// shared marks weights on a filesystem the node did not have to fill —
|
||
// reported so a fabric with a shared filer is not mistaken for one where
|
||
// every node happens to have fetched its own copy.
|
||
bool shared = 3;
|
||
}
|
||
|
||
// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
|
||
//
|
||
// Carried on the heartbeat rather than scraped from the node: the control
|
||
// plane already exports the fleet from one place, and a node exporter per
|
||
// machine would be a scrape target, a port and a firewall rule each.
|
||
message NodeLoad {
|
||
// cpu_percent is the whole machine, 0 to 100 however many cores it has.
|
||
double cpu_percent = 1;
|
||
int32 cpu_count = 2;
|
||
// The kernel's run-queue averages over 1, 5 and 15 minutes.
|
||
double load1 = 3;
|
||
double load5 = 4;
|
||
double load15 = 5;
|
||
// memory_used_bytes is total less available: what the machine could not
|
||
// hand to a new process without evicting something.
|
||
int64 memory_used_bytes = 6;
|
||
int64 memory_available_bytes = 7;
|
||
repeated ShardLoad shards = 8;
|
||
// What the machine's own sensors say; zero is not measured. cpu_temp_c is
|
||
// the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
|
||
// inlet_temp_c the board's inlet or ambient sensor where it has one.
|
||
double cpu_temp_c = 9;
|
||
double cpu_power_w = 10;
|
||
double inlet_temp_c = 11;
|
||
repeated GPUSensor gpus = 12;
|
||
}
|
||
|
||
// GPUSensor is one card's heat, power, and what is holding it back.
|
||
message GPUSensor {
|
||
int32 index = 1;
|
||
double temp_c = 2;
|
||
double memory_temp_c = 3;
|
||
double power_w = 4;
|
||
double power_limit_w = 5;
|
||
// throttle names what is holding the clocks down now: sw_power_cap,
|
||
// hw_slowdown, sw_thermal, hw_thermal, power_brake.
|
||
repeated string throttle = 6;
|
||
// How busy the card is, sampled with the rest, rather than read once at
|
||
// registration. -1 is not reported by the driver. memory_total_mb is set
|
||
// by every agent that sends these, so zero there is an older agent.
|
||
double utilization_percent = 7;
|
||
double memory_utilization_percent = 8;
|
||
double memory_used_mb = 9;
|
||
double memory_total_mb = 10;
|
||
double sm_clock_mhz = 11;
|
||
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
|
||
double ecc_uncorrected = 12;
|
||
// What wears the card out, read every minute; has_wear says these are
|
||
// present. Counts of -1 are not reported for this card.
|
||
bool has_wear = 13;
|
||
int64 pcie_replays = 14;
|
||
int64 retired_pages_sbe = 15;
|
||
int64 retired_pages_dbe = 16;
|
||
bool retirement_pending = 17;
|
||
int64 remapped_rows_correctable = 18;
|
||
int64 remapped_rows_uncorrectable = 19;
|
||
bool remap_pending = 20;
|
||
bool remap_failure = 21;
|
||
int64 nvlink_crc_errors = 22;
|
||
int64 nvlink_replay_errors = 23;
|
||
int64 nvlink_recovery_errors = 24;
|
||
// xid_errors are the driver's error events since the agent started.
|
||
int64 xid_errors = 25;
|
||
int32 last_xid = 26;
|
||
string last_xid_text = 27;
|
||
int64 last_xid_unix = 28;
|
||
}
|
||
|
||
// ShardLoad is one engine process group's use.
|
||
message ShardLoad {
|
||
string shard_id = 1;
|
||
// cpu_cores is in cores, the way top counts a process: 2.5 is two and a
|
||
// half cores busy, which a percentage of the machine would hide on a large
|
||
// one.
|
||
double cpu_cores = 2;
|
||
int64 memory_rss_bytes = 3;
|
||
}
|
||
|
||
// IBPort is one InfiniBand or RoCE port's state and counters.
|
||
message IBPort {
|
||
string hca = 1;
|
||
int32 port = 2;
|
||
// state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged
|
||
// (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT).
|
||
string state = 3;
|
||
string phys_state = 4;
|
||
int32 rate_gbps = 5;
|
||
// link_layer is InfiniBand or Ethernet (RoCE).
|
||
string link_layer = 6;
|
||
// counters are cumulative, by sysfs name; data counters in bytes.
|
||
map<string, uint64> counters = 7;
|
||
|
||
// Identity: the adapter's node GUID and the port's own, as the fabric
|
||
// knows them.
|
||
string node_guid = 8;
|
||
string port_guid = 9;
|
||
// The subnet manager this port sees as master, on InfiniBand. Every port on
|
||
// one physical fabric sees the same one, which is what tells separate
|
||
// fabrics apart when their names are only what somebody typed.
|
||
string sm_guid = 10;
|
||
int32 sm_lid = 11;
|
||
int32 sm_priority = 12;
|
||
string sm_state = 13;
|
||
// sm_error says why sm_guid is empty on an active InfiniBand port:
|
||
// infiniband-diags missing, or no subnet manager answering.
|
||
string sm_error = 14;
|
||
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
||
// subnet manager, and the network is what its fabric is.
|
||
string network = 15;
|
||
string gid_prefix = 16;
|
||
// subnet_managers is every subnet manager the fabric's subnet
|
||
// administrator lists — master and standbys — asked through one port per
|
||
// fabric, so it is on that port only. It is what keeps a fabric one fabric
|
||
// through a failover: the master changes, the list does not.
|
||
repeated SubnetManager subnet_managers = 17;
|
||
string subnet_managers_error = 18;
|
||
}
|
||
|
||
// SubnetManager is one subnet manager on a fabric.
|
||
message SubnetManager {
|
||
string guid = 1;
|
||
int32 lid = 2;
|
||
int32 priority = 3;
|
||
// state is master, standby, discovering or not active.
|
||
string state = 4;
|
||
}
|
||
|
||
// IMEX is the nvidia-imex service and its channels.
|
||
message IMEX {
|
||
// state is systemd's word for it, empty where it is not installed.
|
||
string state = 1;
|
||
int32 channels = 2;
|
||
}
|
||
|
||
// GPUFabricState is one GPU's NVLink domain membership.
|
||
message GPUFabricState {
|
||
int32 index = 1;
|
||
string state = 2;
|
||
string status = 3;
|
||
string cluster_uuid = 4;
|
||
string clique_id = 5;
|
||
string bandwidth = 6;
|
||
}
|
||
|
||
message Heartbeat {
|
||
string node_id = 1;
|
||
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
||
// a card after registration.
|
||
repeated int32 gpu_memory_used_mb = 2;
|
||
repeated string new_problems = 3;
|
||
|
||
// storage for the same reason the GPU memory is here: it changes while the
|
||
// agent runs, and a figure sampled once at registration is a figure that is
|
||
// wrong on exactly the node worth watching — the one busy filling up.
|
||
//
|
||
// Always sent, because it is a few numbers per filesystem.
|
||
repeated Storage storage = 4;
|
||
|
||
// weights only when they changed, because the list is long enough that
|
||
// sending it every few seconds would be paying for nothing most of the time.
|
||
//
|
||
// weights_changed is what says the list is meant: without it an empty list
|
||
// would be ambiguous between "this node now holds nothing" and "nothing to
|
||
// report", and the first would quietly empty a node's inventory.
|
||
bool weights_changed = 5;
|
||
repeated LocalWeights weights = 6;
|
||
|
||
// memory_total_mb for the same reason again: RAM is added to a virtual
|
||
// machine while it runs, or across a reboot the agent does not notice, and a
|
||
// figure read once at registration kept a node that had grown to 32 GB
|
||
// looking like the 7 GB machine it used to be. 0 means not read.
|
||
int64 memory_total_mb = 7;
|
||
|
||
// engine_versions only when they changed, like weights. An engine installed
|
||
// after the agent started is otherwise invisible to placement until
|
||
// somebody restarts the agent — which is a step nobody knows to take.
|
||
bool engines_changed = 8;
|
||
map<string, string> engine_versions = 9;
|
||
|
||
// provision is where installing the node's software stands, sent when it
|
||
// changed. The flag makes an empty list mean "nothing to install" rather
|
||
// than "nothing to say", as for weights.
|
||
bool provision_changed = 10;
|
||
repeated ProvisionStatus provision = 11;
|
||
|
||
// load is how busy the machine is, sampled over the interval since the
|
||
// previous heartbeat. Absent on the first one, which has nothing to take a
|
||
// difference from, and from an agent on a system without /proc.
|
||
NodeLoad load = 12;
|
||
|
||
// ib_ports is every InfiniBand or RoCE port as it stands now, re-read on
|
||
// each heartbeat: a port that goes down after the agent started used to go
|
||
// unnoticed until the agent restarted.
|
||
repeated IBPort ib_ports = 13;
|
||
// fabrics is the fabrics this node can now carry traffic on, sent only when
|
||
// that changed since registration or the last heartbeat that said so — a
|
||
// port going down takes its fabric away, and one coming back returns it.
|
||
bool fabrics_changed = 14;
|
||
repeated FabricAttachment fabrics = 15;
|
||
|
||
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
|
||
// once a minute: a fabric manager restarting drops every card out of its
|
||
// domain, and the node has to stop being offered work that spans it.
|
||
repeated GPUFabricState gpu_fabric = 16;
|
||
IMEX imex = 17;
|
||
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
||
int32 cpu_count = 18;
|
||
|
||
// gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
|
||
// this agent's shards hold on each card, read afresh every heartbeat. What
|
||
// keeps a card out of placement for another tenant is decided from this
|
||
// rather than from registration: a card that had something else on it when
|
||
// the agent started is not taken forever, and one that gains a tenant later
|
||
// is not handed out. Empty from an agent that does not measure it, or from
|
||
// one that could not this time.
|
||
repeated int32 gpu_foreign_memory_mb = 19;
|
||
}
|
||
|
||
// Provision is the software a node should have: which engines, at which
|
||
// versions. Nothing more.
|
||
//
|
||
// Everything else is the node's to work out, because only the node knows it:
|
||
// which build fits its hardware, where that build is published, what its
|
||
// operating system is missing, and whether the result actually runs. The agent
|
||
// converges on this — installs what is absent, checks what is there — and
|
||
// reports each engine through ProvisionStatus. Nothing is taken on trust from
|
||
// what happens to be on the machine: an engine is offered to placement only
|
||
// once the agent has installed it and seen it run.
|
||
message Provision {
|
||
repeated EngineWant engines = 1;
|
||
}
|
||
|
||
// EngineWant names one engine at one version.
|
||
message EngineWant {
|
||
string engine = 1; // "llamacpp"
|
||
string version = 2; // "b10991"
|
||
// variant is the build to install, when one is chosen for the node rather
|
||
// than left to its hardware: "cu13", "cu129". Empty is the agent's choice.
|
||
string variant = 3;
|
||
// offline_base, on a cluster with no way to the internet, is where this
|
||
// engine version's files are on its mirror:
|
||
// http://mirror:8081/_llmgw/engines/llamacpp/b10991. Set, the node never
|
||
// asks the engine's publisher.
|
||
string offline_base = 4;
|
||
// offline_sha256 is each of those files' checksum by name, from the
|
||
// bundle the control plane trusts — not from the mirror, which serves
|
||
// every node over plain HTTP.
|
||
map<string, string> offline_sha256 = 5;
|
||
}
|
||
|
||
// ProvisionStatus is where one engine's installation stands on a node.
|
||
message ProvisionStatus {
|
||
string engine = 1;
|
||
string version = 2;
|
||
string variant = 3;
|
||
// "installing", "ready", "failed", or "unsupported" when no build fits.
|
||
string state = 4;
|
||
// detail says what is happening or what went wrong, in words for a person.
|
||
string detail = 5;
|
||
}
|
||
|
||
// Compute is where an engine runs its model, as it said when it loaded it.
|
||
message Compute {
|
||
// backend is "CUDA", "Vulkan", "Metal" or "CPU".
|
||
string backend = 1;
|
||
// devices are the devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||
repeated string devices = 2;
|
||
// layers_offloaded of layers_total went to the devices; both 0 when the
|
||
// engine did not say.
|
||
int32 layers_offloaded = 3;
|
||
int32 layers_total = 4;
|
||
}
|
||
|
||
// ShardStatus reports a shard's lifecycle transition.
|
||
message ShardStatus {
|
||
string node_id = 1;
|
||
string replica_id = 2;
|
||
string shard_id = 3;
|
||
State state = 4;
|
||
string error = 5;
|
||
int32 exit_code = 6;
|
||
// last_output carries the tail of the engine log on a failure. When a model
|
||
// fails to load this is the only diagnosis available.
|
||
repeated string last_output = 7;
|
||
|
||
// fetch is how far a download has got, when the shard is waiting on one.
|
||
//
|
||
// The agent logs this already, on the node, where nobody is looking. From
|
||
// the control plane a replica pulling a hundred gigabytes and one that is
|
||
// wedged are the same thing — a shard that has been starting for ten
|
||
// minutes — and telling them apart is the difference between waiting and
|
||
// going to look.
|
||
FetchProgress fetch = 8;
|
||
|
||
// compute is what the engine says it computes on, read from its own output
|
||
// as it became ready: sent with STATE_READY when the engine says.
|
||
Compute compute = 9;
|
||
|
||
enum State {
|
||
STATE_UNSPECIFIED = 0;
|
||
STATE_PENDING = 1;
|
||
STATE_STARTING = 2;
|
||
STATE_LOADING = 3;
|
||
STATE_READY = 4;
|
||
STATE_DRAINING = 5;
|
||
STATE_STOPPED = 6;
|
||
STATE_FAILED = 7;
|
||
}
|
||
}
|
||
|
||
// FetchProgress is a download in flight.
|
||
message FetchProgress {
|
||
string source = 1;
|
||
int64 bytes_done = 2;
|
||
// bytes_total is 0 when the server did not say how large the file is, which
|
||
// is a real answer: a percentage would then be invented, and an invented
|
||
// percentage that stops moving is worse than a byte count that keeps rising.
|
||
int64 bytes_total = 3;
|
||
}
|
||
|
||
// StartShard launches one shard. It is idempotent by shard_id: the control
|
||
// plane retries, and a repeat must not produce a second process.
|
||
message StartShard {
|
||
string replica_id = 1;
|
||
string shard_id = 2;
|
||
|
||
int32 rank = 3;
|
||
int32 nnodes = 4;
|
||
string leader_addr = 5;
|
||
int32 leader_port = 6;
|
||
|
||
repeated int32 gpu_indices = 7;
|
||
// hcas are the adapters of the chosen fabric and nothing else. Handing a
|
||
// shard adapters from a second fabric lets NCCL pick one whose peer is
|
||
// unreachable, and the job hangs instead of failing (plan §5.3).
|
||
repeated string hcas = 8;
|
||
string mgmt_interface = 9;
|
||
int32 http_port = 10;
|
||
|
||
EngineProfile profile = 11;
|
||
int64 startup_timeout_ms = 12;
|
||
|
||
// pool and fabric are control plane concepts the agent cannot derive. They
|
||
// are sent so the agent can echo them back in RunningShard after a restart.
|
||
string pool = 13;
|
||
string fabric = 14;
|
||
|
||
// profile_fp identifies the version of the profile, so the agent can say
|
||
// which one a running shard was started from. The agent never interprets it.
|
||
string profile_fp = 15;
|
||
// api_key is what the engine demands of every request, derived by the
|
||
// control plane from the replica id. The router sends it; nothing else
|
||
// that can reach the node's ports has it.
|
||
string api_key = 16;
|
||
// cpu_threads is how many CPU threads a shard with no GPU computes on:
|
||
// the cores placement reserved for it. Zero leaves it to the agent.
|
||
int32 cpu_threads = 17;
|
||
}
|
||
|
||
message EngineProfile {
|
||
string name = 1;
|
||
string engine = 2;
|
||
string gpu_class = 3;
|
||
int32 tp = 4;
|
||
int32 pp = 5;
|
||
int32 gpus_per_node = 6;
|
||
int32 nodes = 7;
|
||
repeated string args = 8;
|
||
map<string, string> env = 9;
|
||
string model_path = 10;
|
||
string served_model_name = 11;
|
||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||
// is the id a request applies one by.
|
||
repeated LoraAdapter adapters = 12;
|
||
// kv_events has a vLLM engine publish its KV cache events to the agent, for
|
||
// the router's precise index.
|
||
bool kv_events = 13;
|
||
}
|
||
|
||
// RemoveWeights deletes cached model files a node no longer needs.
|
||
//
|
||
// The paths are computed by the control plane from the catalog, but the agent
|
||
// checks them again before unlinking anything: it deletes only files inside its
|
||
// own cache directory whose names this system generated. A control plane that
|
||
// is wrong, compromised or simply newer than the agent must not be able to
|
||
// talk it into deleting a model an operator spent a week copying there.
|
||
message RemoveWeights {
|
||
repeated string paths = 1;
|
||
}
|
||
|
||
// StopShard terminates a shard and frees its GPUs.
|
||
message StopShard {
|
||
string shard_id = 1;
|
||
}
|
||
|
||
// DrainShard stops routing to a shard while letting in-flight work finish.
|
||
message DrainShard {
|
||
string shard_id = 1;
|
||
}
|
||
|
||
// Ack confirms a registration or reports that a command could not be accepted.
|
||
message Ack {
|
||
string message_id = 1;
|
||
bool ok = 2;
|
||
string error = 3;
|
||
}
|
||
|
||
// LogsRequest asks a node for the recent lines of one of its logs.
|
||
//
|
||
// Over the stream the agent already holds, because that is the one way into a
|
||
// node every deployment has: the admin page otherwise means ssh through a jump
|
||
// host to a machine whose address nobody remembers, to read a journal only
|
||
// root may open.
|
||
message LogsRequest {
|
||
// request_id pairs the reply with its request; several may be in flight.
|
||
string request_id = 1;
|
||
// source is "agent" for the agent's own log, or a shard id for that
|
||
// engine's output.
|
||
string source = 2;
|
||
int32 lines = 3;
|
||
}
|
||
|
||
// LogsReply answers a LogsRequest.
|
||
message LogsReply {
|
||
string request_id = 1;
|
||
repeated string lines = 2;
|
||
// sources are every log this node can show, so a page can offer them
|
||
// without knowing what runs there.
|
||
repeated string sources = 3;
|
||
// error says why the lines are absent: an unknown source, say.
|
||
string error = 4;
|
||
}
|
||
|
||
// FabricScanRequest asks a node to map the fabric one of its ports is on,
|
||
// with ibnetdiscover: every switch, every adapter and every link between
|
||
// them. One node per fabric is asked, every few minutes; a fabric looks the
|
||
// same from all of its ports.
|
||
message FabricScanRequest {
|
||
string request_id = 1;
|
||
string hca = 2;
|
||
int32 port = 3;
|
||
}
|
||
|
||
// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the
|
||
// control plane, so a format the parser does not yet know is a control plane
|
||
// upgrade rather than one on every node.
|
||
message FabricScanReply {
|
||
string request_id = 1;
|
||
string output = 2;
|
||
string error = 3;
|
||
int64 took_ms = 4;
|
||
}
|
||
|
||
// FabricTestRequest asks a node for one side of an RDMA bandwidth test
|
||
// between two nodes: ib_write_bw, from perftest, over one adapter. The
|
||
// server side is asked first and answers when the test is over; the client
|
||
// dials it. What it proves is what a replica spanning nodes needs and nothing
|
||
// on one node can show: that these two adapters reach each other through the
|
||
// fabric, and how fast.
|
||
message FabricTestRequest {
|
||
string request_id = 1;
|
||
// role is "server" or "client".
|
||
string role = 2;
|
||
string hca = 3;
|
||
int32 port = 4;
|
||
// peer is the server's management address, for the client. perftest
|
||
// exchanges its connection details over TCP before any RDMA.
|
||
string peer = 5;
|
||
// tcp_port is that exchange's port, the same on both sides.
|
||
int32 tcp_port = 6;
|
||
int32 seconds = 7;
|
||
}
|
||
|
||
// FabricTestReply is what one side measured.
|
||
message FabricTestReply {
|
||
string request_id = 1;
|
||
// gbps is the average bandwidth, in gigabits a second.
|
||
double gbps = 2;
|
||
// output is the end of the tool's output, for when the number is not
|
||
// enough.
|
||
string output = 3;
|
||
string error = 4;
|
||
int64 took_ms = 5;
|
||
}
|
||
|
||
// UpgradeAgent tells an agent to replace its own binary with a release and
|
||
// restart.
|
||
//
|
||
// The agent fetches the release for its own platform and checks it against
|
||
// the release's checksums before replacing anything, keeps the old binary
|
||
// beside the new, then exits so systemd starts the new one. Engines keep
|
||
// running throughout, as for any agent restart, and are adopted on the way
|
||
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
||
// no ssh into it.
|
||
message UpgradeAgent {
|
||
string request_id = 1;
|
||
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
||
string version = 2;
|
||
// releases is the base URL release files are downloaded from.
|
||
string releases = 3;
|
||
}
|
||
|
||
// UpgradeReply says whether the new binary is in place. Sent before the
|
||
// agent exits; an empty error means it will be back on the new version.
|
||
message UpgradeReply {
|
||
string request_id = 1;
|
||
string version = 2;
|
||
string error = 3;
|
||
}
|
||
|
||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||
// which the agent fetches as it does the model's weights.
|
||
message LoraAdapter {
|
||
string name = 1;
|
||
string source = 2;
|
||
}
|