CI / test (push) Successful in 1m44s
Not persisting replica state left one hole: restarting the control plane made it forget every replica while the engines kept running. They held GPUs the inventory believed were free, so the next placement failed to allocate, with an error naming the new process rather than the old one. The fix belongs on the agent, not in a database. A row can say "running" about a process that died; the fleet cannot. - the supervisor writes the full shard spec, not just a pid and command line. A pid identifies a process; the spec is what lets it be described back to the control plane and matched to a replica. - Adopt takes over an engine a previous agent left running. It is not this process's child, so it cannot be waited on: it is watched by polling and stopped by signalling its group directly. Its earlier output is gone and the log says so rather than presenting emptiness as silence. - every Register reports what the agent is running, not only the first, because a reconnect may be a control plane restart and has to recover the same way. - the reconciler rebuilds replicas from those reports and reclaims their GPUs. Anything whose model or profile left the catalog is stopped; nothing will route to it and it is holding cards. - an engine the agent cannot identify is killed. A process holding GPUs that nothing can describe is worse than no process, because placement hands those GPUs out again. Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they do not have, and reported them stopped without killing anything. TestAgentReconnects had been passing for no reason. httptest's CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c hijacks the connection and httptest stops tracking hijacked ones — so the agent never reconnected, and the assertion on the connection map passed on a stale entry. Tests now sever connections through their own listener and count opened streams. Plan §11 also records what multi-node still needs, since placement landing early makes it look further along than it is: workers never become Ready because the agent health-checks every shard while only rank 0 serves HTTP; RayCommand is written and tested but never called; and startup is fire-and-forget. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
219 lines
6.6 KiB
Protocol Buffer
219 lines
6.6 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
// Package llmgw.v1 is the protocol between a node agent and the control plane
|
|
// (plan §5.6).
|
|
//
|
|
// The agent dials out and holds a bidirectional stream open. That direction is
|
|
// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
|
|
// control plane does, and an outbound connection needs no inbound rule on every
|
|
// node in the fleet.
|
|
package llmgw.v1;
|
|
|
|
option go_package = "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1;llmgwv1";
|
|
|
|
service AgentService {
|
|
// Connect opens the agent's stream. It stays open for the agent's lifetime;
|
|
// losing it is how each side learns the other is gone.
|
|
rpc Connect(stream AgentMessage) returns (stream ControlMessage);
|
|
}
|
|
|
|
// AgentMessage is anything a node tells the control plane.
|
|
message AgentMessage {
|
|
oneof msg {
|
|
Register register = 1;
|
|
Heartbeat heartbeat = 2;
|
|
ShardStatus shard = 3;
|
|
}
|
|
}
|
|
|
|
// ControlMessage is anything the control plane tells a node.
|
|
message ControlMessage {
|
|
oneof msg {
|
|
StartShard start = 1;
|
|
StopShard stop = 2;
|
|
DrainShard drain = 3;
|
|
Ack ack = 4;
|
|
}
|
|
}
|
|
|
|
// Register is the first message on every stream. It carries the node's full
|
|
// hardware inventory, so the control plane needs no separate discovery call and
|
|
// a reconnecting agent automatically refreshes what the scheduler knows.
|
|
message Register {
|
|
string node_id = 1;
|
|
string hostname = 2;
|
|
// mgmt_addr is the address engines are reachable on, and the address NCCL
|
|
// bootstraps over. Never a fabric address: NCCL rendezvouses over TCP before
|
|
// it moves to RDMA.
|
|
string mgmt_addr = 3;
|
|
map<string, string> labels = 4;
|
|
|
|
repeated GPU gpus = 5;
|
|
// fabrics is empty on a node without InfiniBand. Such a node is its own
|
|
// single-node island and can never host a multi-node replica.
|
|
repeated FabricAttachment fabrics = 6;
|
|
|
|
string agent_version = 7;
|
|
// engine_versions decides which CLI flags and metric names apply; both drift
|
|
// between engine releases (plan §14).
|
|
map<string, string> engine_versions = 8;
|
|
|
|
// running_shards are the engines this agent is already supervising, including
|
|
// ones adopted from a previous agent.
|
|
//
|
|
// This is what lets a control plane restart recover instead of orphaning live
|
|
// engines. Without it the control plane forgets every replica, places new
|
|
// ones, and the old engines keep holding GPUs the inventory believes are free
|
|
// — so the new engines fail to allocate, with an error naming the wrong
|
|
// process (plan §6.5).
|
|
repeated RunningShard running_shards = 9;
|
|
}
|
|
|
|
// RunningShard describes an engine already running on a node.
|
|
//
|
|
// It carries enough for the control plane to rebuild the replica it belongs to,
|
|
// which is more than a shard id: the replica's identity, its pool and fabric,
|
|
// and the leader endpoint clients reach it through.
|
|
message RunningShard {
|
|
string replica_id = 1;
|
|
string shard_id = 2;
|
|
int32 rank = 3;
|
|
int32 nnodes = 4;
|
|
repeated int32 gpu_indices = 5;
|
|
repeated string hcas = 6;
|
|
int32 http_port = 7;
|
|
ShardStatus.State state = 8;
|
|
|
|
// model and profile name the catalog entry this shard was started from.
|
|
string model = 9;
|
|
string profile = 10;
|
|
// pool and fabric are echoed back from the StartShard that created it; the
|
|
// agent has no way to derive them.
|
|
string pool = 11;
|
|
string fabric = 12;
|
|
|
|
string leader_addr = 13;
|
|
int32 leader_port = 14;
|
|
}
|
|
|
|
message GPU {
|
|
int32 index = 1;
|
|
string uuid = 2;
|
|
string name = 3;
|
|
int32 memory_total_mb = 4;
|
|
int32 memory_used_mb = 5;
|
|
string compute_capability = 6;
|
|
int32 numa_node = 7;
|
|
string pci_bus_id = 8;
|
|
string driver_version = 9;
|
|
string cuda_version = 10;
|
|
// nvlink_peers is only populated by agents built with the nvml tag.
|
|
repeated int32 nvlink_peers = 11;
|
|
// problems keeps a GPU out of placement: ECC faults, or memory already held
|
|
// by another process.
|
|
repeated string problems = 12;
|
|
int32 temperature_c = 13;
|
|
int32 ecc_errors = 14;
|
|
}
|
|
|
|
message FabricAttachment {
|
|
// fabric is the operator-assigned name. It is not derived from the GID
|
|
// prefix: OpenSM defaults to fe80:: on every fabric, so a matching prefix
|
|
// proves nothing (plan §5.2).
|
|
string fabric = 1;
|
|
repeated string hcas = 2;
|
|
int32 rate_gbps = 3;
|
|
}
|
|
|
|
// Heartbeat says the node is alive and reports anything that changed cheaply.
|
|
message Heartbeat {
|
|
string node_id = 1;
|
|
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
|
// a card after registration.
|
|
repeated int32 gpu_memory_used_mb = 2;
|
|
repeated string new_problems = 3;
|
|
}
|
|
|
|
// ShardStatus reports a shard's lifecycle transition.
|
|
message ShardStatus {
|
|
string node_id = 1;
|
|
string replica_id = 2;
|
|
string shard_id = 3;
|
|
State state = 4;
|
|
string error = 5;
|
|
int32 exit_code = 6;
|
|
// last_output carries the tail of the engine log on a failure. When a model
|
|
// fails to load this is the only diagnosis available.
|
|
repeated string last_output = 7;
|
|
|
|
enum State {
|
|
STATE_UNSPECIFIED = 0;
|
|
STATE_PENDING = 1;
|
|
STATE_STARTING = 2;
|
|
STATE_LOADING = 3;
|
|
STATE_READY = 4;
|
|
STATE_DRAINING = 5;
|
|
STATE_STOPPED = 6;
|
|
STATE_FAILED = 7;
|
|
}
|
|
}
|
|
|
|
// StartShard launches one shard. It is idempotent by shard_id: the control
|
|
// plane retries, and a repeat must not produce a second process.
|
|
message StartShard {
|
|
string replica_id = 1;
|
|
string shard_id = 2;
|
|
|
|
int32 rank = 3;
|
|
int32 nnodes = 4;
|
|
string leader_addr = 5;
|
|
int32 leader_port = 6;
|
|
|
|
repeated int32 gpu_indices = 7;
|
|
// hcas are the adapters of the chosen fabric and nothing else. Handing a
|
|
// shard adapters from a second fabric lets NCCL pick one whose peer is
|
|
// unreachable, and the job hangs instead of failing (plan §5.3).
|
|
repeated string hcas = 8;
|
|
string mgmt_interface = 9;
|
|
int32 http_port = 10;
|
|
|
|
EngineProfile profile = 11;
|
|
int64 startup_timeout_ms = 12;
|
|
|
|
// pool and fabric are control plane concepts the agent cannot derive. They
|
|
// are sent so the agent can echo them back in RunningShard after a restart.
|
|
string pool = 13;
|
|
string fabric = 14;
|
|
}
|
|
|
|
message EngineProfile {
|
|
string name = 1;
|
|
string engine = 2;
|
|
string gpu_class = 3;
|
|
int32 tp = 4;
|
|
int32 pp = 5;
|
|
int32 gpus_per_node = 6;
|
|
int32 nodes = 7;
|
|
repeated string args = 8;
|
|
map<string, string> env = 9;
|
|
string model_path = 10;
|
|
string served_model_name = 11;
|
|
}
|
|
|
|
// StopShard terminates a shard and frees its GPUs.
|
|
message StopShard {
|
|
string shard_id = 1;
|
|
}
|
|
|
|
// DrainShard stops routing to a shard while letting in-flight work finish.
|
|
message DrainShard {
|
|
string shard_id = 1;
|
|
}
|
|
|
|
// Ack confirms a registration or reports that a command could not be accepted.
|
|
message Ack {
|
|
string message_id = 1;
|
|
bool ok = 2;
|
|
string error = 3;
|
|
}
|