server add gives another machine this llmgw, a certificate, the configuration and store address, lets it into the store through Patroni, and starts it; --vip keeps a floating address on the leader with keepalived, following a new /leader endpoint. Nodes learn every control plane from the leader and notice a dead one by HTTP/2 ping instead of after the kernel's fifteen minutes. The setup, console, node and init screens are drawn full screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
767 lines
29 KiB
Protocol Buffer
767 lines
29 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
// Package llmgw.v1 is the protocol between a node agent and the control plane
|
|
// (plan §5.6).
|
|
//
|
|
// The agent dials out and holds a bidirectional stream open. That direction is
|
|
// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
|
|
// control plane does, and an outbound connection needs no inbound rule on every
|
|
// node in the fleet.
|
|
package llmgw.v1;
|
|
|
|
option go_package = "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1;llmgwv1";
|
|
|
|
service AgentService {
|
|
// Connect opens the agent's stream. It stays open for the agent's lifetime;
|
|
// losing it is how each side learns the other is gone.
|
|
rpc Connect(stream AgentMessage) returns (stream ControlMessage);
|
|
}
|
|
|
|
// AgentMessage is anything a node tells the control plane.
|
|
message AgentMessage {
|
|
oneof msg {
|
|
Register register = 1;
|
|
Heartbeat heartbeat = 2;
|
|
ShardStatus shard = 3;
|
|
LogsReply logs = 4;
|
|
FabricScanReply fabric_scan = 5;
|
|
FabricTestReply fabric_test = 6;
|
|
UpgradeReply upgrade = 7;
|
|
}
|
|
}
|
|
|
|
// ControlMessage is anything the control plane tells a node.
|
|
message ControlMessage {
|
|
oneof msg {
|
|
StartShard start = 1;
|
|
StopShard stop = 2;
|
|
DrainShard drain = 3;
|
|
Ack ack = 4;
|
|
RemoveWeights remove_weights = 5;
|
|
Provision provision = 6;
|
|
LogsRequest logs = 7;
|
|
FabricScanRequest fabric_scan = 8;
|
|
FabricTestRequest fabric_test = 9;
|
|
UpgradeAgent upgrade = 10;
|
|
ControlPlanes control_planes = 11;
|
|
}
|
|
}
|
|
|
|
// ControlPlanes is every control plane an agent may dial, sent by the leader
|
|
// when an agent registers and whenever the list changes. The agent keeps it
|
|
// beside its configuration, so a second control plane added after the node
|
|
// was onboarded is one it fails over to without anybody editing the node.
|
|
message ControlPlanes {
|
|
repeated string servers = 1;
|
|
}
|
|
|
|
// Register is the first message on every stream. It carries the node's full
|
|
// hardware inventory, so the control plane needs no separate discovery call and
|
|
// a reconnecting agent automatically refreshes what the scheduler knows.
|
|
message Register {
|
|
string node_id = 1;
|
|
string hostname = 2;
|
|
// mgmt_addr is the address engines are reachable on, and the address NCCL
|
|
// bootstraps over. Never a fabric address: NCCL rendezvouses over TCP before
|
|
// it moves to RDMA.
|
|
string mgmt_addr = 3;
|
|
map<string, string> labels = 4;
|
|
|
|
repeated GPU gpus = 5;
|
|
// fabrics is empty on a node without InfiniBand. Such a node is its own
|
|
// single-node island and can never host a multi-node replica.
|
|
repeated FabricAttachment fabrics = 6;
|
|
|
|
string agent_version = 7;
|
|
// engine_versions decides which CLI flags and metric names apply; both drift
|
|
// between engine releases (plan §14).
|
|
map<string, string> engine_versions = 8;
|
|
|
|
// running_shards are the engines this agent is already supervising, including
|
|
// ones adopted from a previous agent.
|
|
//
|
|
// This is what lets a control plane restart recover instead of orphaning live
|
|
// engines. Without it the control plane forgets every replica, places new
|
|
// ones, and the old engines keep holding GPUs the inventory believes are free
|
|
// — so the new engines fail to allocate, with an error naming the wrong
|
|
// process (plan §6.5).
|
|
repeated RunningShard running_shards = 9;
|
|
|
|
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
|
// which is what every agent older than this field reports, and is treated as
|
|
// no constraint rather than as a node with no memory.
|
|
//
|
|
// Placement needs it because a CPU profile has no card to be free, so every
|
|
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
|
// would be placed just as happily on a node with 8 GB.
|
|
int64 memory_total_mb = 10;
|
|
|
|
// weights are the model files this node can see, from the directories its
|
|
// configuration names.
|
|
//
|
|
// How they got there is the operator's business and deliberately outside
|
|
// llmgw's: a shared filesystem, an image exported over the fabric, rsync, or
|
|
// a copy made by hand all look identical here. What matters to placement is
|
|
// only whether they are present, because a node without them either cannot
|
|
// start the replica at all or must first move tens of gigabytes.
|
|
//
|
|
// Empty means the node was not asked to look, which is not the same as having
|
|
// nothing — an agent with no model directories configured behaves exactly as
|
|
// every agent did before this existed, and constrains nothing.
|
|
repeated LocalWeights weights = 11;
|
|
|
|
// storage is the disk behind those weights, one entry per filesystem.
|
|
//
|
|
// Reported so an operator can see a node filling up before a fetch dies at
|
|
// ninety percent, and so placement can refuse a node that cannot hold a
|
|
// model rather than discovering it during the download.
|
|
//
|
|
// Empty means the node was not asked to look, exactly as for weights, and
|
|
// constrains nothing.
|
|
repeated Storage storage = 12;
|
|
|
|
// host is what the machine is, so the control plane can have the mirror
|
|
// fetch software built for it rather than assuming every node matches the
|
|
// first.
|
|
Host host = 13;
|
|
|
|
// engines_probed says engine_versions is an answer. Without it an empty map
|
|
// meant "an agent too old to say" and constrained nothing — so a freshly
|
|
// built node with no engine installed yet was a candidate for every model.
|
|
bool engines_probed = 14;
|
|
// cpu_count is the CPUs the agent may run on. Placement reserves cores
|
|
// for CPU replicas out of it, the way it reserves memory out of
|
|
// memory_total_mb; zero, from an older agent, reserves nothing.
|
|
int32 cpu_count = 18;
|
|
// imex is the service that lets a multi-node NVLink domain share memory.
|
|
IMEX imex = 15;
|
|
|
|
// installable_engines are the engines this agent knows how to install
|
|
// itself. The control plane asks it for no others. Empty is an agent older
|
|
// than the field, which could install llama.cpp and nothing else: asked for
|
|
// vLLM, it answered "unsupported", and every node of a cluster half way
|
|
// through an upgrade showed as failing.
|
|
repeated string installable_engines = 16;
|
|
|
|
// features are the requests this agent answers beyond the ones every agent
|
|
// does: "fabric-test". An agent that does not know a request ignores it, so
|
|
// without this the control plane would wait minutes for an answer that is
|
|
// never coming.
|
|
repeated string features = 17;
|
|
}
|
|
|
|
// Host describes the operating system and processor of a node.
|
|
message Host {
|
|
// From /etc/os-release: "ubuntu", "24.04", "noble".
|
|
string os_id = 1;
|
|
string os_version = 2;
|
|
string os_codename = 3;
|
|
// Debian's spelling, which is what package indexes are keyed by: "amd64",
|
|
// "arm64".
|
|
string arch = 4;
|
|
// The vector extensions inference cares about, of those the CPU has. A VM
|
|
// given QEMU's default CPU model has none of them, and runs, and is five
|
|
// times slower, and nothing says why.
|
|
repeated string cpu_flags = 5;
|
|
// nvidia_driver is the driver version, empty when there is none. A card
|
|
// without a driver is not a GPU anything can use.
|
|
string nvidia_driver = 6;
|
|
}
|
|
|
|
// Storage is one filesystem a node keeps weights on.
|
|
//
|
|
// Per filesystem rather than per configured directory: free space belongs to
|
|
// the disk, and two model directories on one disk are one pool of bytes.
|
|
message Storage {
|
|
// path is one directory on this filesystem, so the operator recognises it.
|
|
string path = 1;
|
|
// free_bytes is what a non-root process can write — not the root reserve,
|
|
// which is space the agent cannot have.
|
|
int64 free_bytes = 2;
|
|
int64 total_bytes = 3;
|
|
// shared marks a filesystem the node did not fill. Free space on one is
|
|
// shared with every other node using it, so two nodes each reporting 500 GB
|
|
// free on the same filer have 500 GB between them and not a terabyte.
|
|
bool shared = 4;
|
|
}
|
|
|
|
// RunningShard describes an engine already running on a node.
|
|
//
|
|
// It carries enough for the control plane to rebuild the replica it belongs to,
|
|
// which is more than a shard id: the replica's identity, its pool and fabric,
|
|
// and the leader endpoint clients reach it through.
|
|
message RunningShard {
|
|
string replica_id = 1;
|
|
string shard_id = 2;
|
|
int32 rank = 3;
|
|
int32 nnodes = 4;
|
|
repeated int32 gpu_indices = 5;
|
|
repeated string hcas = 6;
|
|
int32 http_port = 7;
|
|
ShardStatus.State state = 8;
|
|
|
|
// model and profile name the catalog entry this shard was started from.
|
|
string model = 9;
|
|
string profile = 10;
|
|
// pool and fabric are echoed back from the StartShard that created it; the
|
|
// agent has no way to derive them.
|
|
string pool = 11;
|
|
string fabric = 12;
|
|
|
|
string leader_addr = 13;
|
|
int32 leader_port = 14;
|
|
|
|
// profile_fp is the fingerprint of the profile this shard was started from,
|
|
// echoed back from StartShard. The control plane keeps no record of a running
|
|
// replica across its own restart, so without this a shard adopted after a
|
|
// restart could not be compared with the catalog — which is exactly when the
|
|
// catalog is most likely to have been edited.
|
|
string profile_fp = 15;
|
|
}
|
|
|
|
message GPU {
|
|
int32 index = 1;
|
|
string uuid = 2;
|
|
string name = 3;
|
|
int32 memory_total_mb = 4;
|
|
int32 memory_used_mb = 5;
|
|
string compute_capability = 6;
|
|
int32 numa_node = 7;
|
|
string pci_bus_id = 8;
|
|
string driver_version = 9;
|
|
string cuda_version = 10;
|
|
// nvlink_peers is only populated by agents built with the nvml tag.
|
|
repeated int32 nvlink_peers = 11;
|
|
// problems keeps a GPU out of placement: ECC faults, or memory already held
|
|
// by another process.
|
|
repeated string problems = 12;
|
|
int32 temperature_c = 13;
|
|
int32 ecc_errors = 14;
|
|
// utilization_percent is how busy the card was over the last sample period.
|
|
// Zero when the driver is too old to report it, which is not the same as
|
|
// idle: the metric is simply absent rather than wrong.
|
|
int32 utilization_percent = 15;
|
|
// nic_paths is how far each network adapter is from this card, by HCA
|
|
// name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
|
|
map<string, string> nic_paths = 16;
|
|
// The card's multi-node NVLink membership, from nvidia-smi -q: state is
|
|
// Completed once the fabric manager has joined it; cluster_uuid and
|
|
// clique_id name the domain it joined.
|
|
string fabric_state = 17;
|
|
string fabric_status = 18;
|
|
string cluster_uuid = 19;
|
|
string clique_id = 20;
|
|
string fabric_bandwidth = 21;
|
|
}
|
|
|
|
message FabricAttachment {
|
|
// fabric is the operator-assigned name. It is not derived from the GID
|
|
// prefix: OpenSM defaults to fe80:: on every fabric, so a matching prefix
|
|
// proves nothing (plan §5.2).
|
|
string fabric = 1;
|
|
repeated string hcas = 2;
|
|
int32 rate_gbps = 3;
|
|
}
|
|
|
|
// Heartbeat says the node is alive and reports anything that changed cheaply.
|
|
// LocalWeights is one model file or directory a node can see.
|
|
message LocalWeights {
|
|
// path is what an engine would be given, so it can be compared against a
|
|
// model's source without interpretation.
|
|
string path = 1;
|
|
int64 size_bytes = 2;
|
|
// shared marks weights on a filesystem the node did not have to fill —
|
|
// reported so a fabric with a shared filer is not mistaken for one where
|
|
// every node happens to have fetched its own copy.
|
|
bool shared = 3;
|
|
}
|
|
|
|
// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
|
|
//
|
|
// Carried on the heartbeat rather than scraped from the node: the control
|
|
// plane already exports the fleet from one place, and a node exporter per
|
|
// machine would be a scrape target, a port and a firewall rule each.
|
|
message NodeLoad {
|
|
// cpu_percent is the whole machine, 0 to 100 however many cores it has.
|
|
double cpu_percent = 1;
|
|
int32 cpu_count = 2;
|
|
// The kernel's run-queue averages over 1, 5 and 15 minutes.
|
|
double load1 = 3;
|
|
double load5 = 4;
|
|
double load15 = 5;
|
|
// memory_used_bytes is total less available: what the machine could not
|
|
// hand to a new process without evicting something.
|
|
int64 memory_used_bytes = 6;
|
|
int64 memory_available_bytes = 7;
|
|
repeated ShardLoad shards = 8;
|
|
// What the machine's own sensors say; zero is not measured. cpu_temp_c is
|
|
// the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
|
|
// inlet_temp_c the board's inlet or ambient sensor where it has one.
|
|
double cpu_temp_c = 9;
|
|
double cpu_power_w = 10;
|
|
double inlet_temp_c = 11;
|
|
repeated GPUSensor gpus = 12;
|
|
}
|
|
|
|
// GPUSensor is one card's heat, power, and what is holding it back.
|
|
message GPUSensor {
|
|
int32 index = 1;
|
|
double temp_c = 2;
|
|
double memory_temp_c = 3;
|
|
double power_w = 4;
|
|
double power_limit_w = 5;
|
|
// throttle names what is holding the clocks down now: sw_power_cap,
|
|
// hw_slowdown, sw_thermal, hw_thermal, power_brake.
|
|
repeated string throttle = 6;
|
|
// How busy the card is, sampled with the rest, rather than read once at
|
|
// registration. -1 is not reported by the driver. memory_total_mb is set
|
|
// by every agent that sends these, so zero there is an older agent.
|
|
double utilization_percent = 7;
|
|
double memory_utilization_percent = 8;
|
|
double memory_used_mb = 9;
|
|
double memory_total_mb = 10;
|
|
double sm_clock_mhz = 11;
|
|
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
|
|
double ecc_uncorrected = 12;
|
|
// What wears the card out, read every minute; has_wear says these are
|
|
// present. Counts of -1 are not reported for this card.
|
|
bool has_wear = 13;
|
|
int64 pcie_replays = 14;
|
|
int64 retired_pages_sbe = 15;
|
|
int64 retired_pages_dbe = 16;
|
|
bool retirement_pending = 17;
|
|
int64 remapped_rows_correctable = 18;
|
|
int64 remapped_rows_uncorrectable = 19;
|
|
bool remap_pending = 20;
|
|
bool remap_failure = 21;
|
|
int64 nvlink_crc_errors = 22;
|
|
int64 nvlink_replay_errors = 23;
|
|
int64 nvlink_recovery_errors = 24;
|
|
// xid_errors are the driver's error events since the agent started.
|
|
int64 xid_errors = 25;
|
|
int32 last_xid = 26;
|
|
string last_xid_text = 27;
|
|
int64 last_xid_unix = 28;
|
|
}
|
|
|
|
// ShardLoad is one engine process group's use.
|
|
message ShardLoad {
|
|
string shard_id = 1;
|
|
// cpu_cores is in cores, the way top counts a process: 2.5 is two and a
|
|
// half cores busy, which a percentage of the machine would hide on a large
|
|
// one.
|
|
double cpu_cores = 2;
|
|
int64 memory_rss_bytes = 3;
|
|
}
|
|
|
|
// IBPort is one InfiniBand or RoCE port's state and counters.
|
|
message IBPort {
|
|
string hca = 1;
|
|
int32 port = 2;
|
|
// state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged
|
|
// (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT).
|
|
string state = 3;
|
|
string phys_state = 4;
|
|
int32 rate_gbps = 5;
|
|
// link_layer is InfiniBand or Ethernet (RoCE).
|
|
string link_layer = 6;
|
|
// counters are cumulative, by sysfs name; data counters in bytes.
|
|
map<string, uint64> counters = 7;
|
|
|
|
// Identity: the adapter's node GUID and the port's own, as the fabric
|
|
// knows them.
|
|
string node_guid = 8;
|
|
string port_guid = 9;
|
|
// The subnet manager this port sees as master, on InfiniBand. Every port on
|
|
// one physical fabric sees the same one, which is what tells separate
|
|
// fabrics apart when their names are only what somebody typed.
|
|
string sm_guid = 10;
|
|
int32 sm_lid = 11;
|
|
int32 sm_priority = 12;
|
|
string sm_state = 13;
|
|
// sm_error says why sm_guid is empty on an active InfiniBand port:
|
|
// infiniband-diags missing, or no subnet manager answering.
|
|
string sm_error = 14;
|
|
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
|
// subnet manager, and the network is what its fabric is.
|
|
string network = 15;
|
|
string gid_prefix = 16;
|
|
// subnet_managers is every subnet manager the fabric's subnet
|
|
// administrator lists — master and standbys — asked through one port per
|
|
// fabric, so it is on that port only. It is what keeps a fabric one fabric
|
|
// through a failover: the master changes, the list does not.
|
|
repeated SubnetManager subnet_managers = 17;
|
|
string subnet_managers_error = 18;
|
|
}
|
|
|
|
// SubnetManager is one subnet manager on a fabric.
|
|
message SubnetManager {
|
|
string guid = 1;
|
|
int32 lid = 2;
|
|
int32 priority = 3;
|
|
// state is master, standby, discovering or not active.
|
|
string state = 4;
|
|
}
|
|
|
|
// IMEX is the nvidia-imex service and its channels.
|
|
message IMEX {
|
|
// state is systemd's word for it, empty where it is not installed.
|
|
string state = 1;
|
|
int32 channels = 2;
|
|
}
|
|
|
|
// GPUFabricState is one GPU's NVLink domain membership.
|
|
message GPUFabricState {
|
|
int32 index = 1;
|
|
string state = 2;
|
|
string status = 3;
|
|
string cluster_uuid = 4;
|
|
string clique_id = 5;
|
|
string bandwidth = 6;
|
|
}
|
|
|
|
message Heartbeat {
|
|
string node_id = 1;
|
|
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
|
// a card after registration.
|
|
repeated int32 gpu_memory_used_mb = 2;
|
|
repeated string new_problems = 3;
|
|
|
|
// storage for the same reason the GPU memory is here: it changes while the
|
|
// agent runs, and a figure sampled once at registration is a figure that is
|
|
// wrong on exactly the node worth watching — the one busy filling up.
|
|
//
|
|
// Always sent, because it is a few numbers per filesystem.
|
|
repeated Storage storage = 4;
|
|
|
|
// weights only when they changed, because the list is long enough that
|
|
// sending it every few seconds would be paying for nothing most of the time.
|
|
//
|
|
// weights_changed is what says the list is meant: without it an empty list
|
|
// would be ambiguous between "this node now holds nothing" and "nothing to
|
|
// report", and the first would quietly empty a node's inventory.
|
|
bool weights_changed = 5;
|
|
repeated LocalWeights weights = 6;
|
|
|
|
// memory_total_mb for the same reason again: RAM is added to a virtual
|
|
// machine while it runs, or across a reboot the agent does not notice, and a
|
|
// figure read once at registration kept a node that had grown to 32 GB
|
|
// looking like the 7 GB machine it used to be. 0 means not read.
|
|
int64 memory_total_mb = 7;
|
|
|
|
// engine_versions only when they changed, like weights. An engine installed
|
|
// after the agent started is otherwise invisible to placement until
|
|
// somebody restarts the agent — which is a step nobody knows to take.
|
|
bool engines_changed = 8;
|
|
map<string, string> engine_versions = 9;
|
|
|
|
// provision is where installing the node's software stands, sent when it
|
|
// changed. The flag makes an empty list mean "nothing to install" rather
|
|
// than "nothing to say", as for weights.
|
|
bool provision_changed = 10;
|
|
repeated ProvisionStatus provision = 11;
|
|
|
|
// load is how busy the machine is, sampled over the interval since the
|
|
// previous heartbeat. Absent on the first one, which has nothing to take a
|
|
// difference from, and from an agent on a system without /proc.
|
|
NodeLoad load = 12;
|
|
|
|
// ib_ports is every InfiniBand or RoCE port as it stands now, re-read on
|
|
// each heartbeat: a port that goes down after the agent started used to go
|
|
// unnoticed until the agent restarted.
|
|
repeated IBPort ib_ports = 13;
|
|
// fabrics is the fabrics this node can now carry traffic on, sent only when
|
|
// that changed since registration or the last heartbeat that said so — a
|
|
// port going down takes its fabric away, and one coming back returns it.
|
|
bool fabrics_changed = 14;
|
|
repeated FabricAttachment fabrics = 15;
|
|
|
|
// gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
|
|
// once a minute: a fabric manager restarting drops every card out of its
|
|
// domain, and the node has to stop being offered work that spans it.
|
|
repeated GPUFabricState gpu_fabric = 16;
|
|
IMEX imex = 17;
|
|
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
|
int32 cpu_count = 18;
|
|
|
|
// gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
|
|
// this agent's shards hold on each card, read afresh every heartbeat. What
|
|
// keeps a card out of placement for another tenant is decided from this
|
|
// rather than from registration: a card that had something else on it when
|
|
// the agent started is not taken forever, and one that gains a tenant later
|
|
// is not handed out. Empty from an agent that does not measure it, or from
|
|
// one that could not this time.
|
|
repeated int32 gpu_foreign_memory_mb = 19;
|
|
}
|
|
|
|
// Provision is the software a node should have: which engines, at which
|
|
// versions. Nothing more.
|
|
//
|
|
// Everything else is the node's to work out, because only the node knows it:
|
|
// which build fits its hardware, where that build is published, what its
|
|
// operating system is missing, and whether the result actually runs. The agent
|
|
// converges on this — installs what is absent, checks what is there — and
|
|
// reports each engine through ProvisionStatus. Nothing is taken on trust from
|
|
// what happens to be on the machine: an engine is offered to placement only
|
|
// once the agent has installed it and seen it run.
|
|
message Provision {
|
|
repeated EngineWant engines = 1;
|
|
}
|
|
|
|
// EngineWant names one engine at one version.
|
|
message EngineWant {
|
|
string engine = 1; // "llamacpp"
|
|
string version = 2; // "b10991"
|
|
// variant is the build to install, when one is chosen for the node rather
|
|
// than left to its hardware: "cu13", "cu129". Empty is the agent's choice.
|
|
string variant = 3;
|
|
}
|
|
|
|
// ProvisionStatus is where one engine's installation stands on a node.
|
|
message ProvisionStatus {
|
|
string engine = 1;
|
|
string version = 2;
|
|
string variant = 3;
|
|
// "installing", "ready", "failed", or "unsupported" when no build fits.
|
|
string state = 4;
|
|
// detail says what is happening or what went wrong, in words for a person.
|
|
string detail = 5;
|
|
}
|
|
|
|
// ShardStatus reports a shard's lifecycle transition.
|
|
message ShardStatus {
|
|
string node_id = 1;
|
|
string replica_id = 2;
|
|
string shard_id = 3;
|
|
State state = 4;
|
|
string error = 5;
|
|
int32 exit_code = 6;
|
|
// last_output carries the tail of the engine log on a failure. When a model
|
|
// fails to load this is the only diagnosis available.
|
|
repeated string last_output = 7;
|
|
|
|
// fetch is how far a download has got, when the shard is waiting on one.
|
|
//
|
|
// The agent logs this already, on the node, where nobody is looking. From
|
|
// the control plane a replica pulling a hundred gigabytes and one that is
|
|
// wedged are the same thing — a shard that has been starting for ten
|
|
// minutes — and telling them apart is the difference between waiting and
|
|
// going to look.
|
|
FetchProgress fetch = 8;
|
|
|
|
enum State {
|
|
STATE_UNSPECIFIED = 0;
|
|
STATE_PENDING = 1;
|
|
STATE_STARTING = 2;
|
|
STATE_LOADING = 3;
|
|
STATE_READY = 4;
|
|
STATE_DRAINING = 5;
|
|
STATE_STOPPED = 6;
|
|
STATE_FAILED = 7;
|
|
}
|
|
}
|
|
|
|
// FetchProgress is a download in flight.
|
|
message FetchProgress {
|
|
string source = 1;
|
|
int64 bytes_done = 2;
|
|
// bytes_total is 0 when the server did not say how large the file is, which
|
|
// is a real answer: a percentage would then be invented, and an invented
|
|
// percentage that stops moving is worse than a byte count that keeps rising.
|
|
int64 bytes_total = 3;
|
|
}
|
|
|
|
// StartShard launches one shard. It is idempotent by shard_id: the control
|
|
// plane retries, and a repeat must not produce a second process.
|
|
message StartShard {
|
|
string replica_id = 1;
|
|
string shard_id = 2;
|
|
|
|
int32 rank = 3;
|
|
int32 nnodes = 4;
|
|
string leader_addr = 5;
|
|
int32 leader_port = 6;
|
|
|
|
repeated int32 gpu_indices = 7;
|
|
// hcas are the adapters of the chosen fabric and nothing else. Handing a
|
|
// shard adapters from a second fabric lets NCCL pick one whose peer is
|
|
// unreachable, and the job hangs instead of failing (plan §5.3).
|
|
repeated string hcas = 8;
|
|
string mgmt_interface = 9;
|
|
int32 http_port = 10;
|
|
|
|
EngineProfile profile = 11;
|
|
int64 startup_timeout_ms = 12;
|
|
|
|
// pool and fabric are control plane concepts the agent cannot derive. They
|
|
// are sent so the agent can echo them back in RunningShard after a restart.
|
|
string pool = 13;
|
|
string fabric = 14;
|
|
|
|
// profile_fp identifies the version of the profile, so the agent can say
|
|
// which one a running shard was started from. The agent never interprets it.
|
|
string profile_fp = 15;
|
|
// api_key is what the engine demands of every request, derived by the
|
|
// control plane from the replica id. The router sends it; nothing else
|
|
// that can reach the node's ports has it.
|
|
string api_key = 16;
|
|
// cpu_threads is how many CPU threads a shard with no GPU computes on:
|
|
// the cores placement reserved for it. Zero leaves it to the agent.
|
|
int32 cpu_threads = 17;
|
|
}
|
|
|
|
message EngineProfile {
|
|
string name = 1;
|
|
string engine = 2;
|
|
string gpu_class = 3;
|
|
int32 tp = 4;
|
|
int32 pp = 5;
|
|
int32 gpus_per_node = 6;
|
|
int32 nodes = 7;
|
|
repeated string args = 8;
|
|
map<string, string> env = 9;
|
|
string model_path = 10;
|
|
string served_model_name = 11;
|
|
// adapters are LoRA adapters loaded beside the model, in order: the index
|
|
// is the id a request applies one by.
|
|
repeated LoraAdapter adapters = 12;
|
|
}
|
|
|
|
// RemoveWeights deletes cached model files a node no longer needs.
|
|
//
|
|
// The paths are computed by the control plane from the catalog, but the agent
|
|
// checks them again before unlinking anything: it deletes only files inside its
|
|
// own cache directory whose names this system generated. A control plane that
|
|
// is wrong, compromised or simply newer than the agent must not be able to
|
|
// talk it into deleting a model an operator spent a week copying there.
|
|
message RemoveWeights {
|
|
repeated string paths = 1;
|
|
}
|
|
|
|
// StopShard terminates a shard and frees its GPUs.
|
|
message StopShard {
|
|
string shard_id = 1;
|
|
}
|
|
|
|
// DrainShard stops routing to a shard while letting in-flight work finish.
|
|
message DrainShard {
|
|
string shard_id = 1;
|
|
}
|
|
|
|
// Ack confirms a registration or reports that a command could not be accepted.
|
|
message Ack {
|
|
string message_id = 1;
|
|
bool ok = 2;
|
|
string error = 3;
|
|
}
|
|
|
|
// LogsRequest asks a node for the recent lines of one of its logs.
|
|
//
|
|
// Over the stream the agent already holds, because that is the one way into a
|
|
// node every deployment has: the admin page otherwise means ssh through a jump
|
|
// host to a machine whose address nobody remembers, to read a journal only
|
|
// root may open.
|
|
message LogsRequest {
|
|
// request_id pairs the reply with its request; several may be in flight.
|
|
string request_id = 1;
|
|
// source is "agent" for the agent's own log, or a shard id for that
|
|
// engine's output.
|
|
string source = 2;
|
|
int32 lines = 3;
|
|
}
|
|
|
|
// LogsReply answers a LogsRequest.
|
|
message LogsReply {
|
|
string request_id = 1;
|
|
repeated string lines = 2;
|
|
// sources are every log this node can show, so a page can offer them
|
|
// without knowing what runs there.
|
|
repeated string sources = 3;
|
|
// error says why the lines are absent: an unknown source, say.
|
|
string error = 4;
|
|
}
|
|
|
|
// FabricScanRequest asks a node to map the fabric one of its ports is on,
|
|
// with ibnetdiscover: every switch, every adapter and every link between
|
|
// them. One node per fabric is asked, every few minutes; a fabric looks the
|
|
// same from all of its ports.
|
|
message FabricScanRequest {
|
|
string request_id = 1;
|
|
string hca = 2;
|
|
int32 port = 3;
|
|
}
|
|
|
|
// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the
|
|
// control plane, so a format the parser does not yet know is a control plane
|
|
// upgrade rather than one on every node.
|
|
message FabricScanReply {
|
|
string request_id = 1;
|
|
string output = 2;
|
|
string error = 3;
|
|
int64 took_ms = 4;
|
|
}
|
|
|
|
// FabricTestRequest asks a node for one side of an RDMA bandwidth test
|
|
// between two nodes: ib_write_bw, from perftest, over one adapter. The
|
|
// server side is asked first and answers when the test is over; the client
|
|
// dials it. What it proves is what a replica spanning nodes needs and nothing
|
|
// on one node can show: that these two adapters reach each other through the
|
|
// fabric, and how fast.
|
|
message FabricTestRequest {
|
|
string request_id = 1;
|
|
// role is "server" or "client".
|
|
string role = 2;
|
|
string hca = 3;
|
|
int32 port = 4;
|
|
// peer is the server's management address, for the client. perftest
|
|
// exchanges its connection details over TCP before any RDMA.
|
|
string peer = 5;
|
|
// tcp_port is that exchange's port, the same on both sides.
|
|
int32 tcp_port = 6;
|
|
int32 seconds = 7;
|
|
}
|
|
|
|
// FabricTestReply is what one side measured.
|
|
message FabricTestReply {
|
|
string request_id = 1;
|
|
// gbps is the average bandwidth, in gigabits a second.
|
|
double gbps = 2;
|
|
// output is the end of the tool's output, for when the number is not
|
|
// enough.
|
|
string output = 3;
|
|
string error = 4;
|
|
int64 took_ms = 5;
|
|
}
|
|
|
|
// UpgradeAgent tells an agent to replace its own binary with a release and
|
|
// restart.
|
|
//
|
|
// The agent fetches the release for its own platform and checks it against
|
|
// the release's checksums before replacing anything, keeps the old binary
|
|
// beside the new, then exits so systemd starts the new one. Engines keep
|
|
// running throughout, as for any agent restart, and are adopted on the way
|
|
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
|
// no ssh into it.
|
|
message UpgradeAgent {
|
|
string request_id = 1;
|
|
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
|
string version = 2;
|
|
// releases is the base URL release files are downloaded from.
|
|
string releases = 3;
|
|
}
|
|
|
|
// UpgradeReply says whether the new binary is in place. Sent before the
|
|
// agent exits; an empty error means it will be back on the new version.
|
|
message UpgradeReply {
|
|
string request_id = 1;
|
|
string version = 2;
|
|
string error = 3;
|
|
}
|
|
|
|
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
|
// which the agent fetches as it does the model's weights.
|
|
message LoraAdapter {
|
|
string name = 1;
|
|
string source = 2;
|
|
}
|