syntax = "proto3"; // Package llmgw.v1 is the protocol between a node agent and the control plane // (plan §5.6). // // The agent dials out and holds a bidirectional stream open. That direction is // deliberate: GPU nodes sit behind firewalls and NAT far more often than a // control plane does, and an outbound connection needs no inbound rule on every // node in the fleet. package llmgw.v1; option go_package = "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1;llmgwv1"; service AgentService { // Connect opens the agent's stream. It stays open for the agent's lifetime; // losing it is how each side learns the other is gone. rpc Connect(stream AgentMessage) returns (stream ControlMessage); } // AgentMessage is anything a node tells the control plane. message AgentMessage { oneof msg { Register register = 1; Heartbeat heartbeat = 2; ShardStatus shard = 3; LogsReply logs = 4; } } // ControlMessage is anything the control plane tells a node. message ControlMessage { oneof msg { StartShard start = 1; StopShard stop = 2; DrainShard drain = 3; Ack ack = 4; RemoveWeights remove_weights = 5; Provision provision = 6; LogsRequest logs = 7; } } // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. message Register { string node_id = 1; string hostname = 2; // mgmt_addr is the address engines are reachable on, and the address NCCL // bootstraps over. Never a fabric address: NCCL rendezvouses over TCP before // it moves to RDMA. string mgmt_addr = 3; map labels = 4; repeated GPU gpus = 5; // fabrics is empty on a node without InfiniBand. Such a node is its own // single-node island and can never host a multi-node replica. repeated FabricAttachment fabrics = 6; string agent_version = 7; // engine_versions decides which CLI flags and metric names apply; both drift // between engine releases (plan §14). map engine_versions = 8; // running_shards are the engines this agent is already supervising, including // ones adopted from a previous agent. // // This is what lets a control plane restart recover instead of orphaning live // engines. Without it the control plane forgets every replica, places new // ones, and the old engines keep holding GPUs the inventory believes are free // — so the new engines fail to allocate, with an error naming the wrong // process (plan §6.5). repeated RunningShard running_shards = 9; // memory_total_mb is the machine's RAM, and 0 means the agent did not say — // which is what every agent older than this field reports, and is treated as // no constraint rather than as a node with no memory. // // Placement needs it because a CPU profile has no card to be free, so every // ready node looks like a candidate: a 14B model whose working set is 24 GB // would be placed just as happily on a node with 8 GB. int64 memory_total_mb = 10; // weights are the model files this node can see, from the directories its // configuration names. // // How they got there is the operator's business and deliberately outside // llmgw's: a shared filesystem, an image exported over the fabric, rsync, or // a copy made by hand all look identical here. What matters to placement is // only whether they are present, because a node without them either cannot // start the replica at all or must first move tens of gigabytes. // // Empty means the node was not asked to look, which is not the same as having // nothing — an agent with no model directories configured behaves exactly as // every agent did before this existed, and constrains nothing. repeated LocalWeights weights = 11; // storage is the disk behind those weights, one entry per filesystem. // // Reported so an operator can see a node filling up before a fetch dies at // ninety percent, and so placement can refuse a node that cannot hold a // model rather than discovering it during the download. // // Empty means the node was not asked to look, exactly as for weights, and // constrains nothing. repeated Storage storage = 12; // host is what the machine is, so the control plane can have the mirror // fetch software built for it rather than assuming every node matches the // first. Host host = 13; // engines_probed says engine_versions is an answer. Without it an empty map // meant "an agent too old to say" and constrained nothing — so a freshly // built node with no engine installed yet was a candidate for every model. bool engines_probed = 14; } // Host describes the operating system and processor of a node. message Host { // From /etc/os-release: "ubuntu", "24.04", "noble". string os_id = 1; string os_version = 2; string os_codename = 3; // Debian's spelling, which is what package indexes are keyed by: "amd64", // "arm64". string arch = 4; // The vector extensions inference cares about, of those the CPU has. A VM // given QEMU's default CPU model has none of them, and runs, and is five // times slower, and nothing says why. repeated string cpu_flags = 5; // nvidia_driver is the driver version, empty when there is none. A card // without a driver is not a GPU anything can use. string nvidia_driver = 6; } // Storage is one filesystem a node keeps weights on. // // Per filesystem rather than per configured directory: free space belongs to // the disk, and two model directories on one disk are one pool of bytes. message Storage { // path is one directory on this filesystem, so the operator recognises it. string path = 1; // free_bytes is what a non-root process can write — not the root reserve, // which is space the agent cannot have. int64 free_bytes = 2; int64 total_bytes = 3; // shared marks a filesystem the node did not fill. Free space on one is // shared with every other node using it, so two nodes each reporting 500 GB // free on the same filer have 500 GB between them and not a terabyte. bool shared = 4; } // RunningShard describes an engine already running on a node. // // It carries enough for the control plane to rebuild the replica it belongs to, // which is more than a shard id: the replica's identity, its pool and fabric, // and the leader endpoint clients reach it through. message RunningShard { string replica_id = 1; string shard_id = 2; int32 rank = 3; int32 nnodes = 4; repeated int32 gpu_indices = 5; repeated string hcas = 6; int32 http_port = 7; ShardStatus.State state = 8; // model and profile name the catalog entry this shard was started from. string model = 9; string profile = 10; // pool and fabric are echoed back from the StartShard that created it; the // agent has no way to derive them. string pool = 11; string fabric = 12; string leader_addr = 13; int32 leader_port = 14; // profile_fp is the fingerprint of the profile this shard was started from, // echoed back from StartShard. The control plane keeps no record of a running // replica across its own restart, so without this a shard adopted after a // restart could not be compared with the catalog — which is exactly when the // catalog is most likely to have been edited. string profile_fp = 15; } message GPU { int32 index = 1; string uuid = 2; string name = 3; int32 memory_total_mb = 4; int32 memory_used_mb = 5; string compute_capability = 6; int32 numa_node = 7; string pci_bus_id = 8; string driver_version = 9; string cuda_version = 10; // nvlink_peers is only populated by agents built with the nvml tag. repeated int32 nvlink_peers = 11; // problems keeps a GPU out of placement: ECC faults, or memory already held // by another process. repeated string problems = 12; int32 temperature_c = 13; int32 ecc_errors = 14; // utilization_percent is how busy the card was over the last sample period. // Zero when the driver is too old to report it, which is not the same as // idle: the metric is simply absent rather than wrong. int32 utilization_percent = 15; } message FabricAttachment { // fabric is the operator-assigned name. It is not derived from the GID // prefix: OpenSM defaults to fe80:: on every fabric, so a matching prefix // proves nothing (plan §5.2). string fabric = 1; repeated string hcas = 2; int32 rate_gbps = 3; } // Heartbeat says the node is alive and reports anything that changed cheaply. // LocalWeights is one model file or directory a node can see. message LocalWeights { // path is what an engine would be given, so it can be compared against a // model's source without interpretation. string path = 1; int64 size_bytes = 2; // shared marks weights on a filesystem the node did not have to fill — // reported so a fabric with a shared filer is not mistaken for one where // every node happens to have fetched its own copy. bool shared = 3; } message Heartbeat { string node_id = 1; // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on // a card after registration. repeated int32 gpu_memory_used_mb = 2; repeated string new_problems = 3; // storage for the same reason the GPU memory is here: it changes while the // agent runs, and a figure sampled once at registration is a figure that is // wrong on exactly the node worth watching — the one busy filling up. // // Always sent, because it is a few numbers per filesystem. repeated Storage storage = 4; // weights only when they changed, because the list is long enough that // sending it every few seconds would be paying for nothing most of the time. // // weights_changed is what says the list is meant: without it an empty list // would be ambiguous between "this node now holds nothing" and "nothing to // report", and the first would quietly empty a node's inventory. bool weights_changed = 5; repeated LocalWeights weights = 6; // memory_total_mb for the same reason again: RAM is added to a virtual // machine while it runs, or across a reboot the agent does not notice, and a // figure read once at registration kept a node that had grown to 32 GB // looking like the 7 GB machine it used to be. 0 means not read. int64 memory_total_mb = 7; // engine_versions only when they changed, like weights. An engine installed // after the agent started is otherwise invisible to placement until // somebody restarts the agent — which is a step nobody knows to take. bool engines_changed = 8; map engine_versions = 9; // provision is where installing the node's software stands, sent when it // changed. The flag makes an empty list mean "nothing to install" rather // than "nothing to say", as for weights. bool provision_changed = 10; repeated ProvisionStatus provision = 11; } // Provision is the software a node should have: which engines, at which // versions. Nothing more. // // Everything else is the node's to work out, because only the node knows it: // which build fits its hardware, where that build is published, what its // operating system is missing, and whether the result actually runs. The agent // converges on this — installs what is absent, checks what is there — and // reports each engine through ProvisionStatus. Nothing is taken on trust from // what happens to be on the machine: an engine is offered to placement only // once the agent has installed it and seen it run. message Provision { repeated EngineWant engines = 1; } // EngineWant names one engine at one version. message EngineWant { string engine = 1; // "llamacpp" string version = 2; // "b10991" } // ProvisionStatus is where one engine's installation stands on a node. message ProvisionStatus { string engine = 1; string version = 2; string variant = 3; // "installing", "ready", "failed", or "unsupported" when no build fits. string state = 4; // detail says what is happening or what went wrong, in words for a person. string detail = 5; } // ShardStatus reports a shard's lifecycle transition. message ShardStatus { string node_id = 1; string replica_id = 2; string shard_id = 3; State state = 4; string error = 5; int32 exit_code = 6; // last_output carries the tail of the engine log on a failure. When a model // fails to load this is the only diagnosis available. repeated string last_output = 7; // fetch is how far a download has got, when the shard is waiting on one. // // The agent logs this already, on the node, where nobody is looking. From // the control plane a replica pulling a hundred gigabytes and one that is // wedged are the same thing — a shard that has been starting for ten // minutes — and telling them apart is the difference between waiting and // going to look. FetchProgress fetch = 8; enum State { STATE_UNSPECIFIED = 0; STATE_PENDING = 1; STATE_STARTING = 2; STATE_LOADING = 3; STATE_READY = 4; STATE_DRAINING = 5; STATE_STOPPED = 6; STATE_FAILED = 7; } } // FetchProgress is a download in flight. message FetchProgress { string source = 1; int64 bytes_done = 2; // bytes_total is 0 when the server did not say how large the file is, which // is a real answer: a percentage would then be invented, and an invented // percentage that stops moving is worse than a byte count that keeps rising. int64 bytes_total = 3; } // StartShard launches one shard. It is idempotent by shard_id: the control // plane retries, and a repeat must not produce a second process. message StartShard { string replica_id = 1; string shard_id = 2; int32 rank = 3; int32 nnodes = 4; string leader_addr = 5; int32 leader_port = 6; repeated int32 gpu_indices = 7; // hcas are the adapters of the chosen fabric and nothing else. Handing a // shard adapters from a second fabric lets NCCL pick one whose peer is // unreachable, and the job hangs instead of failing (plan §5.3). repeated string hcas = 8; string mgmt_interface = 9; int32 http_port = 10; EngineProfile profile = 11; int64 startup_timeout_ms = 12; // pool and fabric are control plane concepts the agent cannot derive. They // are sent so the agent can echo them back in RunningShard after a restart. string pool = 13; string fabric = 14; // profile_fp identifies the version of the profile, so the agent can say // which one a running shard was started from. The agent never interprets it. string profile_fp = 15; } message EngineProfile { string name = 1; string engine = 2; string gpu_class = 3; int32 tp = 4; int32 pp = 5; int32 gpus_per_node = 6; int32 nodes = 7; repeated string args = 8; map env = 9; string model_path = 10; string served_model_name = 11; } // RemoveWeights deletes cached model files a node no longer needs. // // The paths are computed by the control plane from the catalog, but the agent // checks them again before unlinking anything: it deletes only files inside its // own cache directory whose names this system generated. A control plane that // is wrong, compromised or simply newer than the agent must not be able to // talk it into deleting a model an operator spent a week copying there. message RemoveWeights { repeated string paths = 1; } // StopShard terminates a shard and frees its GPUs. message StopShard { string shard_id = 1; } // DrainShard stops routing to a shard while letting in-flight work finish. message DrainShard { string shard_id = 1; } // Ack confirms a registration or reports that a command could not be accepted. message Ack { string message_id = 1; bool ok = 2; string error = 3; } // LogsRequest asks a node for the recent lines of one of its logs. // // Over the stream the agent already holds, because that is the one way into a // node every deployment has: the admin page otherwise means ssh through a jump // host to a machine whose address nobody remembers, to read a journal only // root may open. message LogsRequest { // request_id pairs the reply with its request; several may be in flight. string request_id = 1; // source is "agent" for the agent's own log, or a shard id for that // engine's output. string source = 2; int32 lines = 3; } // LogsReply answers a LogsRequest. message LogsReply { string request_id = 1; repeated string lines = 2; // sources are every log this node can show, so a page can offer them // without knowing what runs there. repeated string sources = 3; // error says why the lines are absent: an unknown source, say. string error = 4; }