diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index ecf5302..2722542 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13, 0} } // AgentMessage is anything a node tells the control plane. @@ -202,6 +202,7 @@ type ControlMessage struct { // *ControlMessage_Drain // *ControlMessage_Ack // *ControlMessage_RemoveWeights + // *ControlMessage_Provision Msg isControlMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -289,6 +290,15 @@ func (x *ControlMessage) GetRemoveWeights() *RemoveWeights { return nil } +func (x *ControlMessage) GetProvision() *Provision { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_Provision); ok { + return x.Provision + } + } + return nil +} + type isControlMessage_Msg interface { isControlMessage_Msg() } @@ -313,6 +323,10 @@ type ControlMessage_RemoveWeights struct { RemoveWeights *RemoveWeights `protobuf:"bytes,5,opt,name=remove_weights,json=removeWeights,proto3,oneof"` } +type ControlMessage_Provision struct { + Provision *Provision `protobuf:"bytes,6,opt,name=provision,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -323,6 +337,8 @@ func (*ControlMessage_Ack) isControlMessage_Msg() {} func (*ControlMessage_RemoveWeights) isControlMessage_Msg() {} +func (*ControlMessage_Provision) isControlMessage_Msg() {} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. @@ -381,7 +397,15 @@ type Register struct { // // Empty means the node was not asked to look, exactly as for weights, and // constrains nothing. - Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"` + Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"` + // host is what the machine is, so the control plane can have the mirror + // fetch software built for it rather than assuming every node matches the + // first. + Host *Host `protobuf:"bytes,13,opt,name=host,proto3" json:"host,omitempty"` + // engines_probed says engine_versions is an answer. Without it an empty map + // meant "an agent too old to say" and constrained nothing — so a freshly + // built node with no engine installed yet was a candidate for every model. + EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -500,6 +524,113 @@ func (x *Register) GetStorage() []*Storage { return nil } +func (x *Register) GetHost() *Host { + if x != nil { + return x.Host + } + return nil +} + +func (x *Register) GetEnginesProbed() bool { + if x != nil { + return x.EnginesProbed + } + return false +} + +// Host describes the operating system and processor of a node. +type Host struct { + state protoimpl.MessageState `protogen:"open.v1"` + // From /etc/os-release: "ubuntu", "24.04", "noble". + OsId string `protobuf:"bytes,1,opt,name=os_id,json=osId,proto3" json:"os_id,omitempty"` + OsVersion string `protobuf:"bytes,2,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"` + OsCodename string `protobuf:"bytes,3,opt,name=os_codename,json=osCodename,proto3" json:"os_codename,omitempty"` + // Debian's spelling, which is what package indexes are keyed by: "amd64", + // "arm64". + Arch string `protobuf:"bytes,4,opt,name=arch,proto3" json:"arch,omitempty"` + // The vector extensions inference cares about, of those the CPU has. A VM + // given QEMU's default CPU model has none of them, and runs, and is five + // times slower, and nothing says why. + CpuFlags []string `protobuf:"bytes,5,rep,name=cpu_flags,json=cpuFlags,proto3" json:"cpu_flags,omitempty"` + // nvidia_driver is the driver version, empty when there is none. A card + // without a driver is not a GPU anything can use. + NvidiaDriver string `protobuf:"bytes,6,opt,name=nvidia_driver,json=nvidiaDriver,proto3" json:"nvidia_driver,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Host) Reset() { + *x = Host{} + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Host) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Host) ProtoMessage() {} + +func (x *Host) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Host.ProtoReflect.Descriptor instead. +func (*Host) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *Host) GetOsId() string { + if x != nil { + return x.OsId + } + return "" +} + +func (x *Host) GetOsVersion() string { + if x != nil { + return x.OsVersion + } + return "" +} + +func (x *Host) GetOsCodename() string { + if x != nil { + return x.OsCodename + } + return "" +} + +func (x *Host) GetArch() string { + if x != nil { + return x.Arch + } + return "" +} + +func (x *Host) GetCpuFlags() []string { + if x != nil { + return x.CpuFlags + } + return nil +} + +func (x *Host) GetNvidiaDriver() string { + if x != nil { + return x.NvidiaDriver + } + return "" +} + // Storage is one filesystem a node keeps weights on. // // Per filesystem rather than per configured directory: free space belongs to @@ -522,7 +653,7 @@ type Storage struct { func (x *Storage) Reset() { *x = Storage{} - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -534,7 +665,7 @@ func (x *Storage) String() string { func (*Storage) ProtoMessage() {} func (x *Storage) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -547,7 +678,7 @@ func (x *Storage) ProtoReflect() protoreflect.Message { // Deprecated: Use Storage.ProtoReflect.Descriptor instead. func (*Storage) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} } func (x *Storage) GetPath() string { @@ -614,7 +745,7 @@ type RunningShard struct { func (x *RunningShard) Reset() { *x = RunningShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -626,7 +757,7 @@ func (x *RunningShard) String() string { func (*RunningShard) ProtoMessage() {} func (x *RunningShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -639,7 +770,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message { // Deprecated: Use RunningShard.ProtoReflect.Descriptor instead. func (*RunningShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} } func (x *RunningShard) GetReplicaId() string { @@ -776,7 +907,7 @@ type GPU struct { func (x *GPU) Reset() { *x = GPU{} - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -788,7 +919,7 @@ func (x *GPU) String() string { func (*GPU) ProtoMessage() {} func (x *GPU) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -801,7 +932,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message { // Deprecated: Use GPU.ProtoReflect.Descriptor instead. func (*GPU) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} } func (x *GPU) GetIndex() int32 { @@ -923,7 +1054,7 @@ type FabricAttachment struct { func (x *FabricAttachment) Reset() { *x = FabricAttachment{} - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -935,7 +1066,7 @@ func (x *FabricAttachment) String() string { func (*FabricAttachment) ProtoMessage() {} func (x *FabricAttachment) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -948,7 +1079,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead. func (*FabricAttachment) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} } func (x *FabricAttachment) GetFabric() string { @@ -990,7 +1121,7 @@ type LocalWeights struct { func (x *LocalWeights) Reset() { *x = LocalWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1002,7 +1133,7 @@ func (x *LocalWeights) String() string { func (*LocalWeights) ProtoMessage() {} func (x *LocalWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1015,7 +1146,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. func (*LocalWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} } func (x *LocalWeights) GetPath() string { @@ -1070,13 +1201,18 @@ type Heartbeat struct { // somebody restarts the agent — which is a step nobody knows to take. EnginesChanged bool `protobuf:"varint,8,opt,name=engines_changed,json=enginesChanged,proto3" json:"engines_changed,omitempty"` EngineVersions map[string]string `protobuf:"bytes,9,rep,name=engine_versions,json=engineVersions,proto3" json:"engine_versions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // provision is where installing the node's software stands, sent when it + // changed. The flag makes an empty list mean "nothing to install" rather + // than "nothing to say", as for weights. + ProvisionChanged bool `protobuf:"varint,10,opt,name=provision_changed,json=provisionChanged,proto3" json:"provision_changed,omitempty"` + Provision []*ProvisionStatus `protobuf:"bytes,11,rep,name=provision,proto3" json:"provision,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1088,7 +1224,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1101,7 +1237,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *Heartbeat) GetNodeId() string { @@ -1167,6 +1303,206 @@ func (x *Heartbeat) GetEngineVersions() map[string]string { return nil } +func (x *Heartbeat) GetProvisionChanged() bool { + if x != nil { + return x.ProvisionChanged + } + return false +} + +func (x *Heartbeat) GetProvision() []*ProvisionStatus { + if x != nil { + return x.Provision + } + return nil +} + +// Provision is the software a node should have: which engines, at which +// versions. Nothing more. +// +// Everything else is the node's to work out, because only the node knows it: +// which build fits its hardware, where that build is published, what its +// operating system is missing, and whether the result actually runs. The agent +// converges on this — installs what is absent, checks what is there — and +// reports each engine through ProvisionStatus. Nothing is taken on trust from +// what happens to be on the machine: an engine is offered to placement only +// once the agent has installed it and seen it run. +type Provision struct { + state protoimpl.MessageState `protogen:"open.v1"` + Engines []*EngineWant `protobuf:"bytes,1,rep,name=engines,proto3" json:"engines,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Provision) Reset() { + *x = Provision{} + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Provision) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Provision) ProtoMessage() {} + +func (x *Provision) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Provision.ProtoReflect.Descriptor instead. +func (*Provision) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *Provision) GetEngines() []*EngineWant { + if x != nil { + return x.Engines + } + return nil +} + +// EngineWant names one engine at one version. +type EngineWant struct { + state protoimpl.MessageState `protogen:"open.v1"` + Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"` // "llamacpp" + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` // "b10991" + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EngineWant) Reset() { + *x = EngineWant{} + mi := &file_llmgw_v1_agent_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EngineWant) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EngineWant) ProtoMessage() {} + +func (x *EngineWant) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. +func (*EngineWant) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} +} + +func (x *EngineWant) GetEngine() string { + if x != nil { + return x.Engine + } + return "" +} + +func (x *EngineWant) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +// ProvisionStatus is where one engine's installation stands on a node. +type ProvisionStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Variant string `protobuf:"bytes,3,opt,name=variant,proto3" json:"variant,omitempty"` + // "installing", "ready", "failed", or "unsupported" when no build fits. + State string `protobuf:"bytes,4,opt,name=state,proto3" json:"state,omitempty"` + // detail says what is happening or what went wrong, in words for a person. + Detail string `protobuf:"bytes,5,opt,name=detail,proto3" json:"detail,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProvisionStatus) Reset() { + *x = ProvisionStatus{} + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProvisionStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProvisionStatus) ProtoMessage() {} + +func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. +func (*ProvisionStatus) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} +} + +func (x *ProvisionStatus) GetEngine() string { + if x != nil { + return x.Engine + } + return "" +} + +func (x *ProvisionStatus) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *ProvisionStatus) GetVariant() string { + if x != nil { + return x.Variant + } + return "" +} + +func (x *ProvisionStatus) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *ProvisionStatus) GetDetail() string { + if x != nil { + return x.Detail + } + return "" +} + // ShardStatus reports a shard's lifecycle transition. type ShardStatus struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1193,7 +1529,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1205,7 +1541,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1218,7 +1554,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *ShardStatus) GetNodeId() string { @@ -1292,7 +1628,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1304,7 +1640,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1317,7 +1653,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *FetchProgress) GetSource() string { @@ -1373,7 +1709,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1385,7 +1721,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1398,7 +1734,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *StartShard) GetReplicaId() string { @@ -1525,7 +1861,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1537,7 +1873,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1550,7 +1886,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *EngineProfile) GetName() string { @@ -1646,7 +1982,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1658,7 +1994,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1671,7 +2007,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *RemoveWeights) GetPaths() []string { @@ -1691,7 +2027,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1703,7 +2039,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1716,7 +2052,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *StopShard) GetShardId() string { @@ -1736,7 +2072,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1748,7 +2084,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1761,7 +2097,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *DrainShard) GetShardId() string { @@ -1783,7 +2119,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1795,7 +2131,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1808,7 +2144,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *Ack) GetMessageId() string { @@ -1841,14 +2177,15 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" + "\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" + "\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shardB\x05\n" + - "\x03msg\"\x83\x02\n" + + "\x03msg\"\xb8\x02\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ack\x12@\n" + - "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeightsB\x05\n" + - "\x03msg\"\xa7\x05\n" + + "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeights\x123\n" + + "\tprovision\x18\x06 \x01(\v2\x13.llmgw.v1.ProvisionH\x00R\tprovisionB\x05\n" + + "\x03msg\"\xf2\x05\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -1862,13 +2199,24 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x0fmemory_total_mb\x18\n" + " \x01(\x03R\rmemoryTotalMb\x120\n" + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" + - "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x1a9\n" + + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" + + "\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" + + "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"u\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb1\x01\n" + + "\x04Host\x12\x13\n" + + "\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" + + "\n" + + "os_version\x18\x02 \x01(\tR\tosVersion\x12\x1f\n" + + "\vos_codename\x18\x03 \x01(\tR\n" + + "osCodename\x12\x12\n" + + "\x04arch\x18\x04 \x01(\tR\x04arch\x12\x1b\n" + + "\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" + + "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\"u\n" + "\aStorage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + @@ -1925,7 +2273,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" + - "\x06shared\x18\x03 \x01(\bR\x06shared\"\xe2\x03\n" + + "\x06shared\x18\x03 \x01(\bR\x06shared\"\xc8\x04\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -1935,10 +2283,25 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12&\n" + "\x0fmemory_total_mb\x18\a \x01(\x03R\rmemoryTotalMb\x12'\n" + "\x0fengines_changed\x18\b \x01(\bR\x0eenginesChanged\x12P\n" + - "\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x1aA\n" + + "\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x12+\n" + + "\x11provision_changed\x18\n" + + " \x01(\bR\x10provisionChanged\x127\n" + + "\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbb\x03\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + + "\tProvision\x12.\n" + + "\aengines\x18\x01 \x03(\v2\x14.llmgw.v1.EngineWantR\aengines\">\n" + + "\n" + + "EngineWant\x12\x16\n" + + "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" + + "\aversion\x18\x02 \x01(\tR\aversion\"\x8b\x01\n" + + "\x0fProvisionStatus\x12\x16\n" + + "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" + + "\aversion\x18\x02 \x01(\tR\aversion\x12\x18\n" + + "\avariant\x18\x03 \x01(\tR\avariant\x12\x14\n" + + "\x05state\x18\x04 \x01(\tR\x05state\x12\x16\n" + + "\x06detail\x18\x05 \x01(\tR\x06detail\"\xbb\x03\n" + "\vShardStatus\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" + "\n" + @@ -2035,62 +2398,70 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 25) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage (*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage (*Register)(nil), // 3: llmgw.v1.Register - (*Storage)(nil), // 4: llmgw.v1.Storage - (*RunningShard)(nil), // 5: llmgw.v1.RunningShard - (*GPU)(nil), // 6: llmgw.v1.GPU - (*FabricAttachment)(nil), // 7: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights - (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat - (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 11: llmgw.v1.FetchProgress - (*StartShard)(nil), // 12: llmgw.v1.StartShard - (*EngineProfile)(nil), // 13: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 14: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 15: llmgw.v1.StopShard - (*DrainShard)(nil), // 16: llmgw.v1.DrainShard - (*Ack)(nil), // 17: llmgw.v1.Ack - nil, // 18: llmgw.v1.Register.LabelsEntry - nil, // 19: llmgw.v1.Register.EngineVersionsEntry - nil, // 20: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 21: llmgw.v1.EngineProfile.EnvEntry + (*Host)(nil), // 4: llmgw.v1.Host + (*Storage)(nil), // 5: llmgw.v1.Storage + (*RunningShard)(nil), // 6: llmgw.v1.RunningShard + (*GPU)(nil), // 7: llmgw.v1.GPU + (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights + (*Heartbeat)(nil), // 10: llmgw.v1.Heartbeat + (*Provision)(nil), // 11: llmgw.v1.Provision + (*EngineWant)(nil), // 12: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 13: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 14: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 15: llmgw.v1.FetchProgress + (*StartShard)(nil), // 16: llmgw.v1.StartShard + (*EngineProfile)(nil), // 17: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 18: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 19: llmgw.v1.StopShard + (*DrainShard)(nil), // 20: llmgw.v1.DrainShard + (*Ack)(nil), // 21: llmgw.v1.Ack + nil, // 22: llmgw.v1.Register.LabelsEntry + nil, // 23: llmgw.v1.Register.EngineVersionsEntry + nil, // 24: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 25: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 12, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 15, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 16, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 17, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 14, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 18, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 6, // 9: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 7, // 10: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 19, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 5, // 12: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 20, // 18: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 0, // 19: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 11, // 20: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 13, // 21: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 21, // 22: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 23: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 24: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 24, // [24:25] is the sub-list for method output_type - 23, // [23:24] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 10, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 14, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 16, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 19, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 20, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 21, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 18, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 11, // 8: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 22, // 9: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 7, // 10: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 8, // 11: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 23, // 12: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 6, // 13: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 9, // 14: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 5, // 15: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 4, // 16: llmgw.v1.Register.host:type_name -> llmgw.v1.Host + 0, // 17: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 5, // 18: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 19: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 24, // 20: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 13, // 21: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 12, // 22: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 23: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 15, // 24: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 17, // 25: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 25, // 26: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 27: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 28: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 28, // [28:29] is the sub-list for method output_type + 27, // [27:28] is the sub-list for method input_type + 27, // [27:27] is the sub-list for extension type_name + 27, // [27:27] is the sub-list for extension extendee + 0, // [0:27] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -2109,6 +2480,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_Drain)(nil), (*ControlMessage_Ack)(nil), (*ControlMessage_RemoveWeights)(nil), + (*ControlMessage_Provision)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -2116,7 +2488,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 21, + NumMessages: 25, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 1a54e02..79c4ffb 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -34,6 +34,7 @@ message ControlMessage { DrainShard drain = 3; Ack ack = 4; RemoveWeights remove_weights = 5; + Provision provision = 6; } } @@ -101,6 +102,34 @@ message Register { // Empty means the node was not asked to look, exactly as for weights, and // constrains nothing. repeated Storage storage = 12; + + // host is what the machine is, so the control plane can have the mirror + // fetch software built for it rather than assuming every node matches the + // first. + Host host = 13; + + // engines_probed says engine_versions is an answer. Without it an empty map + // meant "an agent too old to say" and constrained nothing — so a freshly + // built node with no engine installed yet was a candidate for every model. + bool engines_probed = 14; +} + +// Host describes the operating system and processor of a node. +message Host { + // From /etc/os-release: "ubuntu", "24.04", "noble". + string os_id = 1; + string os_version = 2; + string os_codename = 3; + // Debian's spelling, which is what package indexes are keyed by: "amd64", + // "arm64". + string arch = 4; + // The vector extensions inference cares about, of those the CPU has. A VM + // given QEMU's default CPU model has none of them, and runs, and is five + // times slower, and nothing says why. + repeated string cpu_flags = 5; + // nvidia_driver is the driver version, empty when there is none. A card + // without a driver is not a GPU anything can use. + string nvidia_driver = 6; } // Storage is one filesystem a node keeps weights on. @@ -234,6 +263,43 @@ message Heartbeat { // somebody restarts the agent — which is a step nobody knows to take. bool engines_changed = 8; map engine_versions = 9; + + // provision is where installing the node's software stands, sent when it + // changed. The flag makes an empty list mean "nothing to install" rather + // than "nothing to say", as for weights. + bool provision_changed = 10; + repeated ProvisionStatus provision = 11; +} + +// Provision is the software a node should have: which engines, at which +// versions. Nothing more. +// +// Everything else is the node's to work out, because only the node knows it: +// which build fits its hardware, where that build is published, what its +// operating system is missing, and whether the result actually runs. The agent +// converges on this — installs what is absent, checks what is there — and +// reports each engine through ProvisionStatus. Nothing is taken on trust from +// what happens to be on the machine: an engine is offered to placement only +// once the agent has installed it and seen it run. +message Provision { + repeated EngineWant engines = 1; +} + +// EngineWant names one engine at one version. +message EngineWant { + string engine = 1; // "llamacpp" + string version = 2; // "b10991" +} + +// ProvisionStatus is where one engine's installation stands on a node. +message ProvisionStatus { + string engine = 1; + string version = 2; + string variant = 3; + // "installing", "ready", "failed", or "unsupported" when no build fits. + string state = 4; + // detail says what is happening or what went wrong, in words for a person. + string detail = 5; } // ShardStatus reports a shard's lifecycle transition. diff --git a/deploy/bootstrap/add-node.sh b/deploy/bootstrap/add-node.sh index 290dd89..5dd2273 100755 --- a/deploy/bootstrap/add-node.sh +++ b/deploy/bootstrap/add-node.sh @@ -108,6 +108,5 @@ echo "== 2/2 onboarding $NAME" echo echo "Check it with: llmgw ctl nodes" echo -echo "A node is not finished until the engines its profiles name are installed" -echo "and the weights are on disk. Onboarding checks that a configured engine" -echo "runs, but it installs neither — see docs/node-onboarding.md." +echo "The agent installs its engines itself; the node takes no model until they" +echo "are installed and seen to run. 'llmgw ctl nodes' shows where that stands." diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh index fdec40a..33790e8 100755 --- a/deploy/bootstrap/onboard-node.sh +++ b/deploy/bootstrap/onboard-node.sh @@ -33,8 +33,8 @@ usage: $0 --node ID --addr HOST --server URL [options] address on the route to --server, found by the agent) --mgmt-if NAME interface NCCL bootstraps over (default: the one holding that address) - --llamacpp-bin P path to llama-server (default: PATH, then /opt/llama.cpp) - --vllm-bin P path to the vllm entry point + --llamacpp-bin P use this llama-server instead of the one the agent installs + --vllm-bin P path to the vllm entry point (not yet installed by the agent) --sglang-python P python interpreter of the SGLang virtualenv --nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized driver installs routinely put it somewhere else. @@ -44,6 +44,8 @@ usage: $0 --node ID --addr HOST --server URL [options] Releases carry static linux/amd64 and linux/arm64 builds; downloading one is better than copying whatever happens to be on the machine you are running this from. + --releases URL where release binaries are downloaded from (default the + project's Gitea releases) --ssh-user NAME (default ilker) USAGE exit 2 @@ -54,6 +56,7 @@ NVIDIA_SMI= IB_ROOT=/sys/class/infiniband LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON= SSH_WAIT=180 PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker +RELEASES=https://gitea.mynodes.xyz/ilker/llmgw/releases/download FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1" while [ $# -gt 0 ]; do @@ -71,6 +74,7 @@ while [ $# -gt 0 ]; do --ib-root) IB_ROOT="$2"; shift 2;; --pki) PKI="$2"; shift 2;; --binary) BINARY="$2"; shift 2;; + --releases) RELEASES="$2"; shift 2;; --ssh-user) SSH_USER="$2"; shift 2;; *) usage;; esac @@ -131,9 +135,28 @@ $SSH "set -e rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem" # 4. The binary, the configuration and the unit. -say "installing the binary" -scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw" -$SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw' +# The node fetches the release CI built for this version, for its own +# architecture, and checks it against the release's checksums — rather than +# being handed whatever binary is on the machine running this script. A build +# that is not a release (a developer's, with commits past the tag) has no such +# file, and is copied as before. +TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')" +if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then + say "installing llmgw $TAG from the release" + $SSH "set -e + arch=\$(dpkg --print-architecture) + name=llmgw-$TAG-linux-\$arch + cd /tmp && rm -f \$name checksums.txt + curl -fsSL -o \$name '$RELEASES/$TAG/'\$name + curl -fsSL -o checksums.txt '$RELEASES/$TAG/checksums.txt' + grep \" \$name\\\$\" checksums.txt | sha256sum -c - >/dev/null + sudo install -m 0755 \$name /usr/local/bin/llmgw + rm -f \$name checksums.txt" +else + say "installing the binary (${TAG:-unversioned}, not a release)" + scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw" + $SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw' +fi say "writing the configuration" NODE_IS_HOSTNAME=no @@ -242,8 +265,9 @@ $SSH 'set -e echo if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then - say "no engine paths given; this node can only run the 'sim' engine until" - say "one is installed and the configuration points at it" + say "the agent now installs the engines the control plane asks for: the" + say "build for this machine, checked against its publisher's checksum and run" + say "before the node is offered any model. 'llmgw ctl nodes' shows progress." fi if [ -n "$ENGINE_PROBLEMS" ]; then echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS" diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml index 5bd2ac5..938d388 100644 --- a/deploy/examples/agent.yaml +++ b/deploy/examples/agent.yaml @@ -84,6 +84,15 @@ discovery: # - /var/lib/llmgw-models # - /mnt/lustre/models +# The agent installs the engines the control plane names (see `engines:` in +# server.yaml) under /opt/llmgw/engines: the build for this machine — CUDA when +# an NVIDIA driver is loaded — checked against its publisher's checksum, with +# the OS packages it needs, and run before the node takes any model. llama.cpp +# is installed this way today. Naming a binary below opts that engine out and +# uses yours instead; vLLM and SGLang still have to be named until the agent +# installs them too. +# engines_dir: /opt/llmgw/engines + engines: vllm: bin: /opt/venvs/vllm/bin/vllm diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 052b291..52ddcfb 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -197,6 +197,13 @@ router: session_load_factor: 1.25 # How aggressively the control plane reconciles. +# The engines every node installs, and at which version. Unset keeps the +# versions this release was checked against (llama.cpp b10991). Each agent +# picks the build for its own hardware and installs it itself; changing a +# version here reaches every node on its next connect. +# engines: +# llamacpp: b10991 + control: interval: 5s base_backoff: 10s diff --git a/docs/reference.md b/docs/reference.md index 57dd1b5..fe3b760 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -54,6 +54,7 @@ rather than repeated. | `adopt_ready_timeout` | `time.Duration` | [Config](#config) | | `tls` | `tlsconf.Config` | [Config](#config) | | `model_dirs` | `[]string` | [Config](#config) | +| `engines_dir` | `string` | [Config](#config) | | `weights_cache` | `string` | [Config](#config) | | `discovery` | `discovery.Options` | [Config](#config) | | `engines` | `engines.Config` | [Config](#config) | @@ -160,6 +161,7 @@ rather than repeated. | `agent_tls` | `tlsconf.Config` | [Server](#server) | | `monitoring` | `Monitoring` | [Server](#server) | | `mirror` | `Mirror` | [Server](#server) | +| `engines` | `map[string]string` | [Server](#server) | | `router` | `Router` | [Server](#server) | | `control` | `Control` | [Server](#server) | | `models` | `[]catalog.Model` | [Server](#server) | @@ -432,6 +434,13 @@ exported over the fabric, local NVMe and a copy made by hand all look identical from here, and placement only needs to know whether the file is present. +### `engines_dir` + +`string` + +EnginesDir is where the agent installs engines, one directory per +build. Defaults to /opt/llmgw/engines. + ### `weights_cache` `string` @@ -589,8 +598,8 @@ know that this page cannot guess. Bin is the llama-server binary. It is the server, not the CLI: the gateway talks OpenAI over HTTP and needs the one that serves it. -Empty looks on PATH, then where release archives are unpacked -(/opt/llama.cpp). +Empty means the agent installs llama.cpp itself, at the version the +control plane asks for. Setting it opts out of that. ### `parallel` @@ -1193,6 +1202,18 @@ Empty means there is none, and /admin/mirror stays unmounted. Takes a [Mirror](#mirror) block. +### `engines` + +`map[string]string` + +Engines are the engines every node installs, and at which version. The +agents install them themselves — the right build for their hardware, +checked against its publisher's checksum and seen to run — so a node is +never trusted to already have what somebody once put there. + +Unset keeps DefaultEngines. A version replaces the default; an empty +one leaves that engine to be installed by hand. + ### `router` `Router` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 730738e..10def64 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -28,6 +28,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" @@ -94,6 +95,10 @@ type Config struct { // is present. ModelDirs []string `koanf:"model_dirs"` + // EnginesDir is where the agent installs engines, one directory per + // build. Defaults to /opt/llmgw/engines. + EnginesDir string `koanf:"engines_dir"` + // WeightsCache is where a remote source is downloaded to. Defaults to the // first entry of ModelDirs, because a node that says where its models live // has already answered this question, and otherwise to @@ -304,16 +309,28 @@ type Agent struct { reg *engines.Registry http connectHTTPClient - // engineOnce probes the installed engines once before registering. + // engineOnce probes the configured engines once before registering. // // Once, because probing SGLang imports it in a Python interpreter — seconds // on a cold cache — and a reconnect is not a reason to pay that again. - // Later changes are found by the slower re-probe the heartbeat starts, not - // by registering again: an engine installed while the agent runs used to - // stay invisible until somebody thought to restart the agent. + // Later changes are found by the slower re-probe the heartbeat starts. engineOnce sync.Once - // engineVers is guarded by mu once the first probe has run. - engineVers map[string]string + // configuredVers are the engines the configuration names and that answer. + // Guarded by mu once the first probe has run. + configuredVers map[string]string + // installedVers are the engines this agent installed and saw run. + // Guarded by mu. + installedVers map[string]string + + // prov installs the engines the control plane asks for. provWants is what + // it asked for, nil until it has; provStatus is where each stands and + // provSent the fingerprint of what the control plane last heard. All + // guarded by mu. provKick starts a pass without waiting for the timer. + prov *provision.Installer + provWants []provision.Want + provStatus map[string]provision.Status + provSent string + provKick chan struct{} // enginesSent is the fingerprint of the engines the control plane last // heard about, from registration or a heartbeat. Guarded by mu. enginesSent string @@ -329,8 +346,9 @@ type Agent struct { fetcher *weights.Fetcher // lastWeights is the fingerprint of the file list most recently sent, so a - // heartbeat carries the list only when it changed. Touched only by the - // heartbeat goroutine, which is the only thing that reads or writes it. + // heartbeat carries the list only when it changed. Guarded by mu: during a + // failover the old stream's heartbeat loop can still be running when the + // new one's starts. lastWeights string // fetchWatchers are the shards waiting on each source, so progress on one @@ -363,12 +381,16 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er if err != nil { return nil, err } - return &Agent{ + a := &Agent{ cfg: cfg, log: log, sup: sup, reg: engines.NewRegistry(cfg.Engines), http: httpClient, shardReplica: map[string]string{}, - }, nil + provStatus: map[string]provision.Status{}, + provKick: make(chan struct{}, 1), + } + a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus} + return a, nil } // Supervisor exposes the process supervisor, for tests and `ctl logs`. @@ -387,6 +409,7 @@ func (a *Agent) Run(ctx context.Context) error { a.log.Error("hardware discovery failed", "err", err) } a.recoverOrphans() + go a.provisionLoop(ctx) attempt := 0 next := 0 @@ -645,9 +668,13 @@ func (a *Agent) scan(ctx context.Context) error { opts := a.cfg.Discovery opts.ModelDirs = a.cfg.ModelDirs inv, err := discovery.Discover(ctx, opts, a.log) + host := discovery.DiscoverHost() a.mu.Lock() a.inv = inv + a.prov.Host = host a.mu.Unlock() + a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch, + "cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver) return err } @@ -685,7 +712,18 @@ func (a *Agent) session(ctx context.Context, server string) error { // engine whether or not its binary exists, so without this a node with no // vLLM accepted a vLLM shard and failed it minutes later. reg.EngineVersions = a.engines(parent) + reg.EnginesProbed = true reg.RunningShards = a.runningShards() + a.mu.Lock() + h := a.prov.Host + // A new stream is a control plane that knows nothing of where installs + // stand, so the next heartbeat tells it again. + a.provSent = "" + a.mu.Unlock() + reg.Host = &llmgwv1.Host{ + OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename, + Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, + } if err := stream.Send(&llmgwv1.AgentMessage{ Msg: &llmgwv1.AgentMessage_Register{Register: reg}, @@ -780,10 +818,22 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { a.reprobeEngines() a.mu.Lock() - if fp := enginesFingerprint(a.engineVers); a.engineVers != nil && fp != a.enginesSent { - a.enginesSent = fp + if vers := a.engineReportLocked(); a.configuredVers != nil && + enginesFingerprint(vers) != a.enginesSent { + a.enginesSent = enginesFingerprint(vers) hb.EnginesChanged = true - hb.EngineVersions = maps.Clone(a.engineVers) + hb.EngineVersions = vers + } + if fp := provisionFingerprint(a.provStatus); fp != a.provSent { + a.provSent = fp + hb.ProvisionChanged = true + for _, e := range slices.Sorted(maps.Keys(a.provStatus)) { + st := a.provStatus[e] + hb.Provision = append(hb.Provision, &llmgwv1.ProvisionStatus{ + Engine: st.Engine, Version: st.Version, Variant: st.Variant, + State: st.State, Detail: st.Detail, + }) + } } a.mu.Unlock() @@ -807,8 +857,12 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { // rather than "nothing to report" — without it, the first heartbeat after // a model was deleted would be indistinguishable from silence. weights := discovery.ScanWeights(a.cfg.ModelDirs) - if fp := weightsFingerprint(weights); fp != a.lastWeights { - a.lastWeights = fp + fp := weightsFingerprint(weights) + a.mu.Lock() + changed := fp != a.lastWeights + a.lastWeights = fp + a.mu.Unlock() + if changed { hb.WeightsChanged = true for _, w := range weights { hb.Weights = append(hb.Weights, &llmgwv1.LocalWeights{ @@ -899,6 +953,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha } case *llmgwv1.ControlMessage_RemoveWeights: a.removeWeights(m.RemoveWeights.GetPaths()) + case *llmgwv1.ControlMessage_Provision: + a.setProvision(m.Provision) case *llmgwv1.ControlMessage_Ack: if !m.Ack.GetOk() { a.log.Warn("control plane rejected a message", @@ -1152,27 +1208,129 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil } -// engines is what this node can run, probed once. -// -// Failures are silence rather than an error: an engine that is not installed is -// the ordinary case on most nodes, and an agent that refused to register over -// one would take a working node out of the fleet for a missing engine it was -// never going to be asked to run. +// engines is what this node can run: what the agent installed and saw run, +// and what the configuration names and answers. Nothing merely found on the +// machine — a llama-server on PATH is somebody's, not necessarily one that +// works, and placing on it is how a replica fails on a node nobody watches. func (a *Agent) engines(ctx context.Context) map[string]string { a.engineOnce.Do(func() { - vers := a.reg.Probe(ctx) + vers := a.reg.ProbeConfigured(ctx) a.mu.Lock() - a.engineVers, a.lastEngineProbe = vers, time.Now() + a.configuredVers, a.lastEngineProbe = vers, time.Now() a.mu.Unlock() - a.log.Info("engines available on this node", - "engines", engines.ProbedNames(vers)) + if len(vers) > 0 { + a.log.Info("engines named in the configuration", + "engines", engines.ProbedNames(vers)) + } }) a.mu.Lock() defer a.mu.Unlock() + vers := a.engineReportLocked() // Whatever registration sends is what the control plane now knows, so a // heartbeat repeats it only once it differs. - a.enginesSent = enginesFingerprint(a.engineVers) - return maps.Clone(a.engineVers) + a.enginesSent = enginesFingerprint(vers) + return vers +} + +// engineReportLocked merges the two sources. Installed wins: it is the one +// that was seen to run. Called with mu held. +func (a *Agent) engineReportLocked() map[string]string { + out := maps.Clone(a.configuredVers) + if out == nil { + out = map[string]string{} + } + maps.Copy(out, a.installedVers) + return out +} + +// provisionEvery is how often installed engines are checked again. A library +// removed by an unrelated upgrade is found here rather than by the next +// replica to start. +const provisionEvery = 10 * time.Minute + +// setProvision takes the control plane's list of wanted engines. +func (a *Agent) setProvision(p *llmgwv1.Provision) { + wants := []provision.Want{} + for _, e := range p.GetEngines() { + wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion()}) + } + a.mu.Lock() + a.provWants = wants + a.mu.Unlock() + select { + case a.provKick <- struct{}{}: + default: + } +} + +// provisionLoop installs and re-checks the wanted engines, one pass at a time. +func (a *Agent) provisionLoop(ctx context.Context) { + t := time.NewTicker(provisionEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-a.provKick: + case <-t.C: + } + a.mu.Lock() + wants := slices.Clone(a.provWants) + managed := a.provWants != nil + a.mu.Unlock() + if !managed { + continue + } + + statuses := a.prov.Converge(ctx, wants) + installed := map[string]string{} + keep := map[string]bool{} + for _, st := range statuses { + keep[st.Engine] = true + if st.State == provision.Ready { + a.reg.UseInstalled(st.Engine, st.Binary) + installed[st.Engine] = st.Version + } + } + a.mu.Lock() + if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) { + a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed)) + } + a.installedVers = installed + // An engine no longer asked for is no longer reported on. + for e := range a.provStatus { + if !keep[e] { + delete(a.provStatus, e) + } + } + a.mu.Unlock() + } +} + +// provisionStatus records a change in one engine's installation. +func (a *Agent) provisionStatus(st provision.Status) { + a.mu.Lock() + prev, had := a.provStatus[st.Engine] + a.provStatus[st.Engine] = st + a.mu.Unlock() + if !had || prev.State != st.State || prev.Detail != st.Detail { + lvl := slog.LevelInfo + if st.State == provision.Failed || st.State == provision.Unsupported { + lvl = slog.LevelWarn + } + a.log.Log(context.Background(), lvl, "engine installation", + "engine", st.Engine, "version", st.Version, "variant", st.Variant, + "state", st.State, "detail", st.Detail) + } +} + +func provisionFingerprint(m map[string]provision.Status) string { + var b strings.Builder + for _, k := range slices.Sorted(maps.Keys(m)) { + st := m[k] + fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail) + } + return b.String() } // engineReprobe is how often the installed engines are looked for again. @@ -1198,14 +1356,14 @@ func (a *Agent) reprobeEngines() { go func() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - vers := a.reg.Probe(ctx) + vers := a.reg.ProbeConfigured(ctx) a.mu.Lock() - changed := enginesFingerprint(vers) != enginesFingerprint(a.engineVers) - a.engineVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false + changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers) + a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false a.mu.Unlock() if changed { - a.log.Info("engines available on this node changed", + a.log.Info("engines named in the configuration changed", "engines", engines.ProbedNames(vers)) } }() diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go new file mode 100644 index 0000000..f0dbccc --- /dev/null +++ b/internal/agent/discovery/host.go @@ -0,0 +1,135 @@ +package discovery + +import ( + "bufio" + "os" + "runtime" + "strings" +) + +// Host is what a node is, as far as installing software on it is concerned. +type Host struct { + // OSID, OSVersion and OSCodename come from /etc/os-release: "ubuntu", + // "24.04", "noble". + OSID string `json:"os_id,omitempty"` + OSVersion string `json:"os_version,omitempty"` + OSCodename string `json:"os_codename,omitempty"` + // Arch is Debian's spelling, "amd64" or "arm64", because that is what + // package names and release archives are keyed by. + Arch string `json:"arch,omitempty"` + // CPUFlags are the vector extensions inference cares about, of those this + // processor has. + CPUFlags []string `json:"cpu_flags,omitempty"` + // NVIDIADriver is the loaded driver's version, empty when there is none. + NVIDIADriver string `json:"nvidia_driver,omitempty"` +} + +// HasCPUFlag reports whether the processor has an extension. +func (h Host) HasCPUFlag(f string) bool { + for _, have := range h.CPUFlags { + if have == f { + return true + } + } + return false +} + +// interestingFlags are the extensions that decide which build of an engine +// runs well. A VM on QEMU's default CPU model has none of them — and still +// runs, five times slower, with nothing saying why. +var interestingFlags = []string{"avx", "avx2", "fma", "f16c", "avx512f", "avx512_bf16", "amx_tile", "asimd", "sve"} + +// DiscoverHost reads what the machine is. +func DiscoverHost() Host { + h := hostFromOSRelease("/etc/os-release") + h.Arch = debianArch(runtime.GOARCH) + h.CPUFlags = cpuFlagsFrom("/proc/cpuinfo") + h.NVIDIADriver = nvidiaDriverFrom("/proc/driver/nvidia/version") + return h +} + +func hostFromOSRelease(path string) Host { + var h Host + f, err := os.Open(path) //nolint:gosec // a fixed system path + if err != nil { + return h + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + k, v, ok := strings.Cut(sc.Text(), "=") + if !ok { + continue + } + v = strings.Trim(v, `"'`) + switch k { + case "ID": + h.OSID = v + case "VERSION_ID": + h.OSVersion = v + case "VERSION_CODENAME": + h.OSCodename = v + } + } + return h +} + +func debianArch(goarch string) string { + switch goarch { + case "amd64", "arm64": + return goarch + case "386": + return "i386" + case "arm": + return "armhf" + } + return goarch +} + +func cpuFlagsFrom(path string) []string { + b, err := os.ReadFile(path) //nolint:gosec // a fixed system path + if err != nil { + return nil + } + have := map[string]bool{} + for _, line := range strings.Split(string(b), "\n") { + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + // "flags" on x86, "Features" on ARM. The first processor is enough: + // a machine does not mix instruction sets across its cores. + if k = strings.TrimSpace(k); k != "flags" && k != "Features" { + continue + } + for _, f := range strings.Fields(v) { + have[f] = true + } + break + } + var out []string + for _, f := range interestingFlags { + if have[f] { + out = append(out, f) + } + } + return out +} + +// nvidiaDriverFrom reads the loaded kernel module's version, which exists +// exactly when a driver is loaded — unlike nvidia-smi, which can be installed +// on a machine whose module failed to load. +func nvidiaDriverFrom(path string) string { + b, err := os.ReadFile(path) //nolint:gosec // a fixed system path + if err != nil { + return "" + } + // "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ..." + fields := strings.Fields(string(b)) + for i, f := range fields { + if f == "Module" && i+1 < len(fields) { + return fields[i+1] + } + } + return "" +} diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 2712be8..145d948 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -151,6 +151,16 @@ func NewRegistry(cfg Config) *Registry { return r } +// UseInstalled points an engine at a build the agent installed, and reports +// whether the engine takes one. +func (r *Registry) UseInstalled(name, path string) bool { + u, ok := r.engines[name].(interface{ UseInstalled(string) }) + if ok { + u.UseInstalled(path) + } + return ok +} + // Add registers an engine, replacing any engine of the same name. func (r *Registry) Add(e Engine) { r.engines[e.Name()] = e } diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 56b2b1b..26f7676 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -3,19 +3,19 @@ package engines import ( "fmt" "os" - "os/exec" "path/filepath" "sort" "strconv" "strings" + "sync/atomic" ) // LlamaCppConfig locates the llama.cpp installation. type LlamaCppConfig struct { // Bin is the llama-server binary. It is the server, not the CLI: the // gateway talks OpenAI over HTTP and needs the one that serves it. - // Empty looks on PATH, then where release archives are unpacked - // (/opt/llama.cpp). + // Empty means the agent installs llama.cpp itself, at the version the + // control plane asks for. Setting it opts out of that. Bin string `koanf:"bin"` // Parallel is how many requests a server handles at once, when a profile // does not say. llama.cpp defaults to one, which would make every second @@ -29,7 +29,10 @@ type LlamaCppConfig struct { // small model on CPU at a speed that is fine for testing the whole path, and it // serves the same OpenAI API as the others, so nothing downstream can tell the // difference. -type LlamaCpp struct{ cfg LlamaCppConfig } +type LlamaCpp struct { + cfg LlamaCppConfig + installed atomic.Pointer[string] +} // Name identifies the engine in the catalog. func (*LlamaCpp) Name() string { return "llamacpp" } @@ -170,33 +173,25 @@ func ggufIn(path string) (string, error) { return filepath.Join(path, names[0]), nil } -// llamaServerPlaces are where llama.cpp's release archives are unpacked when -// they are not on PATH. The archive carries its shared libraries beside the -// binary, so it is usually left in one directory rather than installed. -var llamaServerPlaces = []string{ - "/opt/llama.cpp/llama-server", - "/opt/llama.cpp/bin/llama-server", - "/opt/llama.cpp/build/bin/llama-server", - "/usr/local/llama.cpp/llama-server", -} +// UseInstalled points the engine at a build the agent installed and checked. +// It takes precedence over everything else, because it is the only one that +// was seen to run. +func (l *LlamaCpp) UseInstalled(path string) { l.installed.Store(&path) } -// bin is the llama-server to run: the configured one, else the one on PATH, -// else the first of the usual unpack locations that exists. -// -// Found rather than required in the configuration, because every node here -// had the same path written into its file by hand, and a node set up without -// it reported no engine at all. +// Configured reports whether the agent's configuration names a binary. A +// named binary is the operator's deliberate choice; one merely found on PATH +// is not, and is not reported as installed. +func (l *LlamaCpp) Configured() bool { return l.cfg.Bin != "" } + +// bin is the llama-server to run: the installed build, else the configured +// one, else whatever PATH has — which is only ever reached by a node not +// provisioned by the agent and not told where its engine is. func (l *LlamaCpp) bin() string { + if p := l.installed.Load(); p != nil { + return *p + } if l.cfg.Bin != "" { return l.cfg.Bin } - if p, err := exec.LookPath("llama-server"); err == nil { - return p - } - for _, p := range llamaServerPlaces { - if info, err := os.Stat(p); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { - return p - } - } return "llama-server" } diff --git a/internal/agent/engines/probe.go b/internal/agent/engines/probe.go index 51edd80..15a0ebb 100644 --- a/internal/agent/engines/probe.go +++ b/internal/agent/engines/probe.go @@ -157,11 +157,49 @@ func firstVersion(s string) string { return Unknown } +// Configured is an engine whose location the agent's configuration names. +type Configured interface { + Configured() bool +} + +// ProbeConfigured probes only the engines the configuration names. +// +// Probe looks everywhere — PATH, the default interpreter — and reports what +// it finds, which is exactly the trust in whatever happens to be on a machine +// that installing engines replaces. A node the agent provisions reports what +// it installed, plus what the operator named on purpose; nothing else. +func (r *Registry) ProbeConfigured(ctx context.Context) map[string]string { + out := map[string]string{} + for name, e := range r.engines { + c, ok := e.(Configured) + if !ok || !c.Configured() { + continue + } + p, ok := e.(Prober) + if !ok { + continue + } + if v, err := p.Probe(ctx); err == nil { + if v == "" { + v = Unknown + } + out[name] = v + } + } + return out +} + // Probe reports whether the llama.cpp server is installed. func (l *LlamaCpp) Probe(context.Context) (string, error) { return installedBinary(l.bin()) } +// Configured reports whether the configuration names vLLM. +func (v *VLLM) Configured() bool { return v.cfg.Bin != "" } + +// Configured reports whether the configuration names SGLang's interpreter. +func (g *SGLang) Configured() bool { return g.cfg.Python != "" } + // Probe reports whether vLLM is installed. func (v *VLLM) Probe(context.Context) (string, error) { bin := v.cfg.Bin diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go new file mode 100644 index 0000000..2c11357 --- /dev/null +++ b/internal/agent/provision/install.go @@ -0,0 +1,534 @@ +package provision + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// Want is one engine at one version. +type Want struct { + Engine string + Version string +} + +// The states an engine's installation can be in. +const ( + Installing = "installing" + Ready = "ready" + Failed = "failed" + Unsupported = "unsupported" +) + +// Status is where one engine's installation stands. +type Status struct { + Engine string + Version string + Variant string + State string + // Detail is for a person: what is happening, or what went wrong. + Detail string + // Binary is the executable, once ready. + Binary string +} + +// Installer installs engines under one directory. +type Installer struct { + // Root is where engines are installed, one directory per build: + // //-. + Root string + Host discovery.Host + Log *slog.Logger + HTTP *http.Client + // OnStatus hears every change, so a download that takes minutes is seen + // to be under way rather than silent. + OnStatus func(Status) + + // Run executes a command; a seam for tests. Nil runs it for real. + Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error) + // APIBase and DownloadBase default to GitHub; tests point them at a fake. + APIBase string + DownloadBase string +} + +// DefaultRoot is where engines are installed when nothing says otherwise. +const DefaultRoot = "/opt/llmgw/engines" + +// marker is written into a build's directory once it has been installed and +// seen to run. Its absence means the directory is not to be trusted. +const marker = ".llmgw-installed" + +type markerFile struct { + Engine string `json:"engine"` + Version string `json:"version"` + Variant string `json:"variant"` + Binary string `json:"binary"` + SHA256 map[string]string `json:"sha256"` + Installed time.Time `json:"installed"` +} + +// Converge brings every wanted engine to ready, or says why it cannot. +// +// An engine already installed is checked again rather than believed: its +// directory can be damaged, or a library it needs removed by somebody's +// upgrade, and that is found here rather than by the next replica to fail. +func (in *Installer) Converge(ctx context.Context, wants []Want) []Status { + out := make([]Status, 0, len(wants)) + for _, w := range wants { + st := in.one(ctx, w) + in.report(st) + out = append(out, st) + } + return out +} + +func (in *Installer) one(ctx context.Context, w Want) Status { + st := Status{Engine: w.Engine, Version: w.Version} + recipe, ok := Recipes[w.Engine] + if !ok { + st.State = Unsupported + st.Detail = "this agent does not know how to install " + w.Engine + + "; install it by hand and name it in the agent's configuration" + return st + } + build, err := recipe.Choose(w.Version, in.Host) + if err != nil { + st.State, st.Detail = Unsupported, err.Error() + return st + } + st.Variant = build.Variant + dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant) + + // Installed before: check it still runs, and trust nothing else about it. + if _, err := os.Stat(filepath.Join(dir, marker)); err == nil { + // The packages are checked too. Removing one is exactly the kind of + // change an unrelated upgrade makes, and ldd below would catch it only + // as a missing library with no hint of which package held it. + if err := in.packages(ctx, build.Packages, &st); err != nil { + st.State, st.Detail = Failed, err.Error() + return st + } + bin, version, err := in.verify(ctx, dir, build) + if err == nil { + return in.ready(st, build, bin, version) + } + in.log().Warn("an installed engine no longer runs; installing it again", + "engine", w.Engine, "version", w.Version, "err", err) + } + + if err := in.install(ctx, recipe, w, build, dir, &st); err != nil { + st.State, st.Detail, st.Binary = Failed, err.Error(), "" + return st + } + bin, version, err := in.verify(ctx, dir, build) + if err != nil { + st.State, st.Detail = Failed, err.Error() + return st + } + return in.ready(st, build, bin, version) +} + +func (in *Installer) ready(st Status, b Build, bin, version string) Status { + st.State, st.Binary = Ready, bin + st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version) + var missing []string + for _, f := range b.CPUFlags { + if !in.Host.HasCPUFlag(f) { + missing = append(missing, f) + } + } + if len(missing) > 0 { + // Said on every report, not once in a log: it is the difference between + // five tokens a second and forty, and a VM given QEMU's default CPU + // model looks entirely healthy otherwise. + st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") + + ", so it will run several times slower (a VM needs CPU type \"host\")" + } + return st +} + +// install fetches, checks and unpacks a build into dir. +// +// Into a scratch directory renamed into place at the end, so a failure halfway +// leaves nothing that looks installed. +func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error { + if err := in.packages(ctx, b.Packages, st); err != nil { + return err + } + + in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version) + sums, err := in.checksums(ctx, r.Repo, w.Version) + if err != nil { + return err + } + + tmp := dir + ".partial" + _ = os.RemoveAll(tmp) + if err := os.MkdirAll(tmp, 0o755); err != nil { + return err + } + defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed + + got := map[string]string{} + for _, asset := range b.Assets { + want, ok := sums[asset] + if !ok { + // Refused rather than installed unchecked: this is a binary that + // will run as root on every node. + return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified", + w.Engine, w.Version, asset) + } + archive := filepath.Join(tmp, asset+".download") + if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil { + return err + } + in.progress(st, "unpacking %s", asset) + if err := untarGz(archive, tmp); err != nil { + return fmt.Errorf("unpacking %s: %w", asset, err) + } + _ = os.Remove(archive) + got[asset] = want + } + + in.progress(st, "checking that %s runs", b.Binary) + bin, _, err := in.verify(ctx, tmp, b) + if err != nil { + return err + } + rel, _ := filepath.Rel(tmp, bin) + m, _ := json.MarshalIndent(markerFile{ + Engine: w.Engine, Version: w.Version, Variant: b.Variant, + Binary: rel, SHA256: got, Installed: time.Now().UTC(), + }, "", " ") + if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret + return err + } + _ = os.RemoveAll(dir) + if err := os.Rename(tmp, dir); err != nil { + return err + } + in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version, + "variant", b.Variant, "dir", dir) + return nil +} + +// verify finds the binary and runs it. +// +// Two checks, because they fail differently. ldd names a missing library +// outright, where running the binary would only say it could not start. And +// running it is the only proof that the rest — the right instruction set, the +// right driver — is in place. +func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) { + bin, err = findFile(dir, b.Binary) + if err != nil { + return "", "", err + } + if out, err := in.run(ctx, nil, "ldd", bin); err == nil { + var missing []string + for _, line := range strings.Split(string(out), "\n") { + if strings.Contains(line, "not found") { + missing = append(missing, strings.TrimSpace(strings.SplitN(line, "=>", 2)[0])) + } + } + if len(missing) > 0 { + return "", "", fmt.Errorf("%s cannot load %s", b.Binary, strings.Join(missing, ", ")) + } + } + tctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + out, err := in.run(tctx, nil, bin, "--version") + if err != nil { + return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out)) + } + return bin, firstVersionLine(out), nil +} + +// packages installs the operating system packages a build needs. +// +// Only those missing, and from the node's own package sources: a package the +// base image already has is not touched, so an engine never upgrades +// something the rest of the machine depends on. +func (in *Installer) packages(ctx context.Context, pkgs []string, st *Status) error { + var missing []string + for _, p := range pkgs { + out, err := in.run(ctx, nil, "dpkg-query", "-W", "-f=${db:Status-Abbrev}", p) + if err != nil || !strings.HasPrefix(string(out), "ii") { + missing = append(missing, p) + } + } + if len(missing) == 0 { + return nil + } + in.progress(st, "installing %s", strings.Join(missing, ", ")) + env := []string{"DEBIAN_FRONTEND=noninteractive"} + // A lock timeout rather than failing on the lock: a machine a minute old is + // running cloud-init's and unattended-upgrades' own apt, and the first + // attempt on every new node would otherwise fail. + lock := "-o=DPkg::Lock::Timeout=600" + if out, err := in.run(ctx, env, "apt-get", lock, "update"); err != nil { + return fmt.Errorf("apt-get update: %v: %s", err, lastLine(out)) + } + args := append([]string{lock, "install", "-y", "--no-install-recommends"}, missing...) + if out, err := in.run(ctx, env, "apt-get", args...); err != nil { + return fmt.Errorf("installing %s: %v: %s", strings.Join(missing, ", "), err, lastLine(out)) + } + return nil +} + +// checksums reads the sha256 GitHub publishes for each asset of a release. +func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) { + url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := in.client().Do(req) + if err != nil { + return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err) + } + defer resp.Body.Close() + switch { + case resp.StatusCode == http.StatusNotFound: + return nil, fmt.Errorf("%s has no release %s", repo, tag) + case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests: + // Unauthenticated callers get sixty requests an hour per address, and + // a whole fleet behind one NAT shares them. + return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status) + case resp.StatusCode != http.StatusOK: + return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag) + } + var rel struct { + Assets []struct { + Name string `json:"name"` + Digest string `json:"digest"` + } `json:"assets"` + } + if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil { + return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err) + } + out := map[string]string{} + for _, a := range rel.Assets { + if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 { + out[a.Name] = sum + } + } + return out, nil +} + +func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error { + url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := in.client().Do(req) + if err != nil { + return fmt.Errorf("downloading %s: %w", asset, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading %s: %s", asset, resp.Status) + } + if resp.ContentLength > 0 { + in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20) + } else { + in.progress(st, "downloading %s", asset) + } + f, err := os.Create(dst) //nolint:gosec // inside the install root + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(io.MultiWriter(f, h), resp.Body) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return fmt.Errorf("downloading %s: %w", asset, err) + } + if got := hex.EncodeToString(h.Sum(nil)); got != want { + return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)", + asset, got[:12], want[:12]) + } + return nil +} + +// untarGz unpacks an archive under dir, refusing anything that would land +// outside it. A release archive is somebody else's file, and a path of +// "../../etc/cron.d/x" in it would otherwise be written as root. +func untarGz(archive, dir string) error { + f, err := os.Open(archive) //nolint:gosec // inside the install root + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + target, err := within(dir, h.Name) + if err != nil { + return err + } + switch h.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above + if err != nil { + return err + } + if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum + out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + case tar.TypeSymlink: + // Libraries are versioned through symlinks, and those are + // relative. One pointing anywhere else is refused. + if filepath.IsAbs(h.Linkname) { + return fmt.Errorf("%s links outside the archive", h.Name) + } + if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil { + return fmt.Errorf("%s links outside the archive", h.Name) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + _ = os.Remove(target) + if err := os.Symlink(h.Linkname, target); err != nil { + return err + } + default: + // Devices, FIFOs and hard links have no business in a release. + } + } +} + +func within(dir, name string) (string, error) { + target := filepath.Join(dir, name) //nolint:gosec // checked on the next line + rel, err := filepath.Rel(dir, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%s would unpack outside the install directory", name) + } + return target, nil +} + +func findFile(dir, name string) (string, error) { + var found string + _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || found != "" { + return nil //nolint:nilerr // an unreadable subtree is not the binary + } + if !d.IsDir() && d.Name() == name { + found = p + } + return nil + }) + if found == "" { + return "", fmt.Errorf("the build has no %s in it", name) + } + return found, nil +} + +func (in *Installer) progress(st *Status, format string, args ...any) { + st.State, st.Detail = Installing, fmt.Sprintf(format, args...) + in.report(*st) +} + +func (in *Installer) report(st Status) { + if in.OnStatus != nil { + in.OnStatus(st) + } +} + +func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) { + if in.Run != nil { + return in.Run(ctx, env, name, args...) + } + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed + cmd.Env = append(os.Environ(), env...) + return cmd.CombinedOutput() +} + +func (in *Installer) root() string { + if in.Root != "" { + return in.Root + } + return DefaultRoot +} + +func (in *Installer) log() *slog.Logger { + if in.Log != nil { + return in.Log + } + return slog.Default() +} + +func (in *Installer) client() *http.Client { + if in.HTTP != nil { + return in.HTTP + } + return &http.Client{Timeout: 30 * time.Minute} +} + +func (in *Installer) apiBase() string { + if in.APIBase != "" { + return in.APIBase + } + return "https://api.github.com" +} + +func (in *Installer) downloadBase() string { + if in.DownloadBase != "" { + return in.DownloadBase + } + return "https://github.com" +} + +func lastLine(b []byte) string { + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + return strings.TrimSpace(lines[len(lines)-1]) +} + +func firstVersionLine(b []byte) string { + for _, l := range strings.Split(string(b), "\n") { + if strings.Contains(strings.ToLower(l), "version") { + return strings.TrimSpace(l) + } + } + return "" +} diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go new file mode 100644 index 0000000..70808f4 --- /dev/null +++ b/internal/agent/provision/install_test.go @@ -0,0 +1,254 @@ +package provision + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +var ubuntu = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64", CPUFlags: []string{"avx2"}} + +// release is a fake GitHub: one release, its assets and their digests. +type release struct { + assets map[string][]byte + digests map[string]string + gets int + mu sync.Mutex +} + +func tarGz(t *testing.T, files map[string]string, links map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, body := range files { + _ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte(body)) + } + for name, target := range links { + _ = tw.WriteHeader(&tar.Header{Name: name, Linkname: target, Typeflag: tar.TypeSymlink}) + } + _ = tw.Close() + _ = gz.Close() + return buf.Bytes() +} + +func (r *release) serve(t *testing.T) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + r.mu.Lock() + r.gets++ + r.mu.Unlock() + if strings.HasPrefix(req.URL.Path, "/repos/") { + var out struct { + Assets []map[string]string `json:"assets"` + } + for name := range r.assets { + out.Assets = append(out.Assets, map[string]string{"name": name, "digest": r.digests[name]}) + } + _ = json.NewEncoder(w).Encode(out) + return + } + body, ok := r.assets[filepath.Base(req.URL.Path)] + if !ok { + http.NotFound(w, req) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +func newRelease(assets map[string][]byte) *release { + r := &release{assets: assets, digests: map[string]string{}} + for name, b := range assets { + sum := sha256.Sum256(b) + r.digests[name] = "sha256:" + hex.EncodeToString(sum[:]) + } + return r +} + +// fakeSystem answers dpkg-query, apt-get, ldd and the engine itself. +type fakeSystem struct { + mu sync.Mutex + installed map[string]bool + aptRuns int + lddOut string + calls []string +} + +func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, name+" "+strings.Join(args, " ")) + switch { + case name == "dpkg-query": + if f.installed[args[len(args)-1]] { + return []byte("ii "), nil + } + return []byte("un "), errors.New("exit 1") + case name == "apt-get": + f.aptRuns++ + if args[1] == "install" { + for _, p := range args[4:] { + f.installed[p] = true + } + } + return nil, nil + case name == "ldd": + return []byte(f.lddOut), nil + case strings.HasSuffix(name, "llama-server"): + return []byte("version: 10991 (abc)\n"), nil + } + return nil, errors.New("unexpected command " + name) +} + +func installer(t *testing.T, srv *httptest.Server, sys *fakeSystem, host discovery.Host) (*Installer, *[]Status) { + var seen []Status + var mu sync.Mutex + return &Installer{ + Root: t.TempDir(), Host: host, Run: sys.run, + APIBase: srv.URL, DownloadBase: srv.URL, HTTP: srv.Client(), + OnStatus: func(s Status) { mu.Lock(); seen = append(seen, s); mu.Unlock() }, + }, &seen +} + +func llamaRelease(t *testing.T) *release { + return newRelease(map[string][]byte{ + "llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t, + map[string]string{"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libllama.so.0": "x"}, + map[string]string{"llama-b1/libllama.so": "libllama.so.0"}), + }) +} + +func TestAnEngineIsInstalledFromNothing(t *testing.T) { + // A node rebuilt from a bare image: no engine, no OpenMP runtime. What it + // ends with has to have come from the publisher, checked, and been run. + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{}} + in, seen := installer(t, rel.serve(t), sys, ubuntu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if len(got) != 1 || got[0].State != Ready { + t.Fatalf("not ready: %+v", got) + } + if !sys.installed["libgomp1"] { + t.Error("the OpenMP runtime the build links against was not installed") + } + if !strings.HasPrefix(got[0].Binary, in.Root) { + t.Errorf("binary %s is not the one installed under %s", got[0].Binary, in.Root) + } + if _, err := os.Lstat(filepath.Join(filepath.Dir(got[0].Binary), "libllama.so")); err != nil { + t.Errorf("the library symlink was not unpacked: %v", err) + } + var states []string + for _, s := range *seen { + states = append(states, s.State+": "+s.Detail) + } + if !strings.Contains(strings.Join(states, "\n"), "downloading") { + t.Errorf("a download was not reported while it ran:\n%s", strings.Join(states, "\n")) + } +} + +func TestAnInstalledEngineIsCheckedNotFetchedAgain(t *testing.T) { + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{}} + in, _ := installer(t, rel.serve(t), sys, ubuntu) + w := []Want{{Engine: "llamacpp", Version: "b1"}} + in.Converge(t.Context(), w) + before := rel.gets + + got := in.Converge(t.Context(), w) + if got[0].State != Ready { + t.Fatalf("second pass: %+v", got[0]) + } + if rel.gets != before { + t.Errorf("a ready engine was downloaded again (%d requests)", rel.gets-before) + } +} + +func TestAnEngineThatCannotLoadALibraryIsNotReady(t *testing.T) { + // Found on a node before: llama-server present, executable, and unable to + // start for want of a library — while the node reported ready. + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}, + lddOut: "\tlibcuda.so.1 => not found\n"} + in, _ := installer(t, rel.serve(t), sys, ubuntu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "libcuda.so.1") { + t.Fatalf("expected a failure naming the library: %+v", got[0]) + } + if _, err := os.Stat(filepath.Join(in.Root, "llamacpp", "b1-cpu", marker)); err == nil { + t.Error("a build that does not run was marked installed") + } +} + +func TestADownloadThatDoesNotMatchItsChecksumIsRefused(t *testing.T) { + rel := llamaRelease(t) + for name := range rel.digests { + rel.digests[name] = "sha256:" + strings.Repeat("0", 64) + } + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}} + in, _ := installer(t, rel.serve(t), sys, ubuntu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") { + t.Fatalf("a tampered download was not refused: %+v", got[0]) + } +} + +func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) { + rel := newRelease(map[string][]byte{ + "llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t, + map[string]string{"../../escaped": "x", "llama-server": ""}, nil), + }) + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}} + in, _ := installer(t, rel.serve(t), sys, ubuntu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Failed { + t.Fatalf("an archive escaping its directory was installed: %+v", got[0]) + } +} + +func TestTheBuildFollowsTheHardware(t *testing.T) { + gpu := ubuntu + gpu.NVIDIADriver = "570.86.15" + b, err := chooseLlamaCpp("b1", gpu) + if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 { + t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err) + } + if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "rocky", Arch: "amd64"}); err == nil { + t.Error("a build was chosen for a system llama.cpp does not publish for") + } + + slow := ubuntu + slow.CPUFlags = nil + st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "") + if !strings.Contains(st.Detail, "avx2") { + t.Errorf("a CPU without AVX2 was not called out: %q", st.Detail) + } +} + +func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) { + in := &Installer{Root: t.TempDir(), Host: ubuntu} + got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.29.0"}}) + if got[0].State != Unsupported { + t.Errorf("got %+v", got[0]) + } +} diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go new file mode 100644 index 0000000..0b9910e --- /dev/null +++ b/internal/agent/provision/recipe.go @@ -0,0 +1,110 @@ +// Package provision installs the engines a node is asked to have. +// +// The control plane says which engines at which versions; everything else is +// worked out here, on the node, because only the node knows it: which build +// fits its processor and driver, what its operating system is missing, and +// whether the result actually runs. Nothing already on the machine is taken +// on trust. An engine counts as installed when this package installed it, +// checked every file against the checksum its publisher gave, and watched it +// start. +package provision + +import ( + "fmt" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// Build is one published build of an engine, chosen for one machine. +type Build struct { + // Variant names the build: "cpu", "cuda-12.8". + Variant string + // Assets are the release files to fetch, unpacked in order into one + // directory. + Assets []string + // Packages are operating system packages the build links against and the + // base image does not carry. + Packages []string + // Binary is the executable's file name, found anywhere in the unpacked + // tree. + Binary string + // CPUFlags are extensions the build is slow without. Missing ones are + // reported, not refused: slow is still running. + CPUFlags []string +} + +// Recipe knows how to install one engine. +type Recipe struct { + // Repo is the GitHub repository whose releases carry the builds. + Repo string + // Choose picks the build for a machine, or says why none fits. + Choose func(version string, h discovery.Host) (Build, error) +} + +// Recipes are the engines this agent can install. +// +// vLLM and SGLang are not here yet: they are Python environments of several +// gigabytes rather than an archive, and are installed by hand and named in the +// agent's configuration until they are. +var Recipes = map[string]Recipe{ + "llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp}, +} + +// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also +// carries a matching cudart archive, so the node needs a driver and nothing +// else — no toolkit. +const cudaForLlamaCpp = "12.8" + +func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { + // The published Linux builds are built on Ubuntu, and are what llama.cpp + // tests. Another distribution may well run them, but finding out on a + // production node is not the place to learn it. + if h.OSID != "ubuntu" { + return Build{}, fmt.Errorf("llama.cpp publishes builds for Ubuntu, and this node runs %s", + orUnknown(h.OSID)) + } + var arch string + switch h.Arch { + case "amd64": + arch = "x64" + case "arm64": + arch = "arm64" + default: + return Build{}, fmt.Errorf("llama.cpp publishes no Ubuntu build for %s", orUnknown(h.Arch)) + } + + b := Build{ + Variant: "cpu", + Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)}, + // Everything else the build links is in the archive or in any Ubuntu + // install; OpenMP's runtime is the one a cloud image lacks, and the + // server does not start without it. + Packages: []string{"libgomp1"}, + Binary: "llama-server", + } + if arch == "x64" { + // The archive carries a backend per CPU generation and loads the best + // one, down to SSE4.2. Below AVX2 it runs, several times slower. + b.CPUFlags = []string{"avx2"} + } + // The CUDA build is published for x64 at this version; on arm64 it is a + // newer CUDA that the driver may not support, so an arm64 GPU node runs + // the CPU build until someone decides otherwise. + if h.NVIDIADriver != "" && arch == "x64" { + name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp) + b.Variant = "cuda-" + cudaForLlamaCpp + b.Assets = []string{ + fmt.Sprintf("llama-%s-%s", version, name), + fmt.Sprintf("cudart-llama-%s-%s", version, name), + } + } + return b, nil +} + +func orUnknown(s string) string { + if strings.TrimSpace(s) == "" { + return "an unrecognised system" + } + return s +} diff --git a/internal/agent/selfdescribe_test.go b/internal/agent/selfdescribe_test.go index 41afc71..5919ce4 100644 --- a/internal/agent/selfdescribe_test.go +++ b/internal/agent/selfdescribe_test.go @@ -1,9 +1,11 @@ package agent import ( + "context" "os" "slices" "testing" + "time" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" ) @@ -60,15 +62,15 @@ func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) { engines := func(m *llmgwv1.AgentMessage) *llmgwv1.Heartbeat { return m.GetHeartbeat() } a.mu.Lock() - a.engineVers = map[string]string{"llamacpp": "unknown"} - a.enginesSent = enginesFingerprint(a.engineVers) + a.configuredVers = map[string]string{"llamacpp": "unknown"} + a.enginesSent = enginesFingerprint(a.configuredVers) a.mu.Unlock() if hb := engines(a.heartbeat()); hb.GetEnginesChanged() { t.Errorf("unchanged engines were resent") } a.mu.Lock() - a.engineVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"} + a.configuredVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"} a.mu.Unlock() hb := engines(a.heartbeat()) if !hb.GetEnginesChanged() || len(hb.GetEngineVersions()) != 2 { @@ -78,3 +80,40 @@ func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) { t.Errorf("reported again on the next heartbeat") } } + +func TestWhatTheControlPlaneAsksToInstallIsReportedBack(t *testing.T) { + cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), + EnginesDir: t.TempDir()} + cfg.Superv.WorkDir = t.TempDir() + a, err := New(cfg, nil, nil) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.provisionLoop(ctx) + + // An engine this agent cannot install: no network is needed to learn so, + // and the answer has to reach the control plane rather than a log. + a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{{Engine: "vllm", Version: "0.29.0"}}}) + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + hb := a.heartbeat().GetHeartbeat() + if hb.GetProvisionChanged() { + p := hb.GetProvision() + if len(p) != 1 || p[0].GetState() != "unsupported" { + t.Fatalf("reported %v", p) + } + a.mu.Lock() + _, reported := a.engineReportLocked()["vllm"] + a.mu.Unlock() + if reported { + t.Error("an engine that was not installed is reported as present") + } + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the installation's outcome never reached a heartbeat") +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index e2988ca..219a6d0 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -189,13 +189,27 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { _ = w.Flush() // Unhealthy GPUs are usually the answer when a node looks idle but - // nothing schedules onto it. + // nothing schedules onto it. An engine still installing, or one + // that failed to, is the other usual answer. + said := false for _, n := range nodes { for _, p := range n.Unhealthy { fmt.Printf("\n%s: %s (kept out of placement)", n.ID, p) + said = true + } + for _, p := range n.Provision { + if p.State == "ready" { + continue + } + fmt.Printf("\n%s: %s %s %s: %s", n.ID, p.Engine, p.Version, p.State, p.Detail) + said = true + } + for _, w := range n.Warnings { + fmt.Printf("\n%s: %s", n.ID, w) + said = true } } - if hasUnhealthy(nodes) { + if said { fmt.Println() } return nil @@ -478,12 +492,3 @@ func dash(s string) string { } return s } - -func hasUnhealthy(nodes []adminapi.NodeView) bool { - for _, n := range nodes { - if len(n.Unhealthy) > 0 { - return true - } - } - return false -} diff --git a/internal/cli/server.go b/internal/cli/server.go index f9ca2a4..a145d56 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -401,6 +401,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live } } svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log) + svc.SetEngines(cfg.WantedEngines()) + log.Info("nodes install these engines", "engines", cfg.WantedEngines()) // A standby must refuse agents as well as refusing to place. Commands // travel down the stream the agent opened, so a standby holding one is a // node the leader cannot command — and the leader's placements would fail diff --git a/internal/config/config.go b/internal/config/config.go index c1ef662..a2e9fd6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -72,6 +72,15 @@ type Server struct { // Empty means there is none, and /admin/mirror stays unmounted. Mirror Mirror `koanf:"mirror"` + // Engines are the engines every node installs, and at which version. The + // agents install them themselves — the right build for their hardware, + // checked against its publisher's checksum and seen to run — so a node is + // never trusted to already have what somebody once put there. + // + // Unset keeps DefaultEngines. A version replaces the default; an empty + // one leaves that engine to be installed by hand. + Engines map[string]string `koanf:"engines"` + Router Router `koanf:"router"` Control Control `koanf:"control"` Models []catalog.Model `koanf:"models"` @@ -542,3 +551,25 @@ func (s Server) ControlPlaneMode() bool { } return false } + +// DefaultEngines are the engine versions this release was checked against: +// `llmgw ctl engine-check` accepted every flag llmgw builds for them. +var DefaultEngines = map[string]string{ + "llamacpp": "b10991", +} + +// WantedEngines is DefaultEngines with the configuration laid over it. +func (s Server) WantedEngines() map[string]string { + out := map[string]string{} + for k, v := range DefaultEngines { + out[k] = v + } + for k, v := range s.Engines { + if v == "" { + delete(out, k) + continue + } + out[k] = v + } + return out +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0cba22e..a75eba8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -182,3 +182,20 @@ func TestAModelWithProfilesStillMakesAControlPlane(t *testing.T) { t.Error("a declared model with profiles is still a control plane") } } + +func TestEnginesDefaultToTheCheckedVersions(t *testing.T) { + var s Server + if got := s.WantedEngines()["llamacpp"]; got != DefaultEngines["llamacpp"] { + t.Errorf("unset config wants llamacpp %q", got) + } + s.Engines = map[string]string{"llamacpp": "b11000", "vllm": "0.29.0"} + got := s.WantedEngines() + if got["llamacpp"] != "b11000" || got["vllm"] != "0.29.0" { + t.Errorf("configured versions not taken: %v", got) + } + // Empty leaves an engine to be installed by hand. + s.Engines = map[string]string{"llamacpp": ""} + if _, ok := s.WantedEngines()["llamacpp"]; ok { + t.Error("an emptied engine is still wanted") + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 3c5cc18..d22c645 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -13,6 +13,7 @@ import ( "io" "net/http" "slices" + "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/bench" @@ -78,6 +79,15 @@ type NodeView struct { // which is not the same as having none — placement treats it that way too, // or a control plane upgrade would empty a fleet. Engines []string `json:"engines,omitempty"` + // Provision is where installing each wanted engine stands. An engine is in + // Engines only once its installation here is ready. + Provision []inventory.EngineInstall `json:"provision,omitempty"` + // OS is the operating system and architecture, as the agent read them. + OS string `json:"os,omitempty"` + // Warnings are facts about the machine that make it slow or unusable + // without making it unhealthy: a VM with no vector instructions runs every + // model, at a fifth of the speed. + Warnings []string `json:"warnings,omitempty"` // Models are the weights this node can see. Absent means it was not asked // to look, which is not the same as holding none. @@ -230,6 +240,14 @@ func (a *API) Nodes() []NodeView { v.Engines = append(v.Engines, name) } slices.Sort(v.Engines) + v.Provision = n.Provision + if h := n.Host; h.OSID != "" { + v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch) + if h.Arch == "amd64" && !h.HasCPUFlag("avx2") { + v.Warnings = append(v.Warnings, "no AVX2: inference runs several times slower "+ + "(a VM needs CPU type \"host\")") + } + } for _, w := range n.Weights { v.Models = append(v.Models, w.Path) } diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 6094599..0fe0e1c 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -23,8 +23,15 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { State: inventory.Ready, AgentVersion: r.GetAgentVersion(), EngineVersions: r.GetEngineVersions(), + EnginesProbed: r.GetEnginesProbed(), MemoryTotalMB: int(r.GetMemoryTotalMb()), } + if h := r.GetHost(); h != nil { + n.Host = discovery.Host{ + OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(), + Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(), + } + } for _, w := range r.GetWeights() { n.Weights = append(n.Weights, discovery.LocalWeights{ Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(), diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index e8678d8..db94ca0 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -49,8 +49,14 @@ type Service struct { mu sync.RWMutex conns map[string]*conn + + // engines is what every node is told to install, set once at start. + engines map[string]string } +// SetEngines sets the engines every node installs, by name and version. +func (s *Service) SetEngines(e map[string]string) { s.engines = maps.Clone(e) } + // conn is one agent's open stream. type conn struct { nodeID string @@ -157,6 +163,21 @@ func (s *Service) Connect( return err } + // What this node should have installed. Sent on every registration, so a + // node rebuilt from nothing is told the same as one that has been here a + // month, and a changed version reaches every node on its next connect. + if s.engines != nil { + p := &llmgwv1.Provision{} + for _, e := range slices.Sorted(maps.Keys(s.engines)) { + p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: s.engines[e]}) + } + if err := stream.Send(&llmgwv1.ControlMessage{ + Msg: &llmgwv1.ControlMessage_Provision{Provision: p}, + }); err != nil { + return err + } + } + // Recover replica state from what this node is already running, before any // reconciliation pass can conclude the cluster is empty and place // duplicates (plan §6.5). @@ -272,7 +293,8 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { storage := hb.GetStorage() memMB := int(hb.GetMemoryTotalMb()) if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && - !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() { + !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() && + !hb.GetProvisionChanged() { return } @@ -339,6 +361,30 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { } n.EngineVersions = vers } + if hb.GetProvisionChanged() { + prev := map[string]inventory.EngineInstall{} + for _, p := range n.Provision { + prev[p.Engine] = p + } + n.Provision = n.Provision[:0] + for _, p := range hb.GetProvision() { + e := inventory.EngineInstall{ + Engine: p.GetEngine(), Version: p.GetVersion(), Variant: p.GetVariant(), + State: p.GetState(), Detail: p.GetDetail(), + } + // A state change is worth a line; a download's progress is not. + if old, ok := prev[e.Engine]; !ok || old.State != e.State { + lvl := slog.LevelInfo + if e.State == "failed" || e.State == "unsupported" { + lvl = slog.LevelWarn + } + s.log.Log(context.Background(), lvl, "engine installation on a node", + "node", nodeID, "engine", e.Engine, "version", e.Version, + "variant", e.Variant, "state", e.State, "detail", e.Detail) + } + n.Provision = append(n.Provision, e) + } + } s.inv.Upsert(n) } diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index ece7595..4617286 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -102,6 +102,24 @@ type Node struct { // version decides which metric names and CLI flags apply (plan §14). AgentVersion string `json:"agent_version,omitempty"` EngineVersions map[string]string `json:"engine_versions,omitempty"` + // EnginesProbed says EngineVersions is an answer, so an empty map means + // "has none" rather than "an agent too old to say". + EnginesProbed bool `json:"engines_probed,omitempty"` + + // Host is the operating system and processor, as the agent reads them. + Host discovery.Host `json:"host"` + // Provision is where installing each wanted engine stands on this node. + Provision []EngineInstall `json:"provision,omitempty"` +} + +// EngineInstall is where one engine's installation stands on a node. +type EngineInstall struct { + Engine string `json:"engine"` + Version string `json:"version"` + Variant string `json:"variant,omitempty"` + // State is "installing", "ready", "failed" or "unsupported". + State string `json:"state"` + Detail string `json:"detail,omitempty"` } // HasFabric reports whether the node is attached to a fabric. diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 60af93b..b10d776 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -403,9 +403,14 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje // A node that reports no engines at all is not excluded. That is what // every agent older than this reports, and emptying a fleet on a // control plane upgrade is a worse failure than the one being fixed. - if p.Engine != "" && len(n.EngineVersions) > 0 { + // + // An agent that says it probed is believed when it reports nothing: + // that is a node still installing, or one where installing failed, + // and either way not a place to start an engine. sim is built into + // the agent and never installed. + if p.Engine != "" && p.Engine != "sim" && (n.EnginesProbed || len(n.EngineVersions) > 0) { if _, ok := n.EngineVersions[p.Engine]; !ok { - reject("does not have " + p.Engine + " installed") + reject("does not have " + p.Engine + " installed" + installing(n, p.Engine)) continue } } @@ -526,3 +531,13 @@ func (s *Scheduler) claimedMemory(nodeID string) int { } return total } + +// installing says why an engine is missing, when the node has said. +func installing(n inventory.Node, engine string) string { + for _, p := range n.Provision { + if p.Engine == engine && p.Detail != "" { + return " (" + p.State + ": " + p.Detail + ")" + } + } + return "" +} diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 13f7f0e..9a241d6 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -797,3 +797,33 @@ func TestAProfileThatDeclaresNothingClaimsNothing(t *testing.T) { t.Errorf("an undeclared neighbour should not fill the node:\n%s", res.Explain()) } } + +func TestANodeStillInstallingItsEngineIsNotACandidate(t *testing.T) { + // A node rebuilt from nothing registers before its agent has installed + // anything, and reports no engines. Treated as "too old to say", it was a + // candidate for every model at the moment it could run none of them. + fresh := node("fresh", "h100-80g", 8) + fresh.EnginesProbed = true + fresh.Provision = []inventory.EngineInstall{{ + Engine: "vllm", Version: "0.29.0", State: "installing", Detail: "downloading", + }} + + res := New(store(fresh)).Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement != nil { + t.Fatal("placed on a node whose engine is not installed yet") + } + if !strings.Contains(res.Explain(), "installing: downloading") { + t.Errorf("the rejection should say the engine is on its way:\n%s", res.Explain()) + } + + // The simulated engine is part of the agent, so it is never installed and + // never missing. + res = New(store(fresh)).Place([]Profile{{ + Name: "sim", Engine: "sim", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement == nil { + t.Errorf("sim was refused:\n%s", res.Explain()) + } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index ef796c8..045dce3 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -101,7 +101,7 @@ requested whenever a fallback served it.

Nodes

{{if .Nodes}} - +
NodeStateGPUsRAMDiskClassesFabricsModelsReplicasLast seen
{{range .Nodes}} @@ -111,12 +111,15 @@ requested whenever a fallback served it.

second, and an em dash where nothing was reported: an agent that was never asked about its disk must not read as a full one. */}} + -{{if .Unhealthy}}{{end}} +{{if .Unhealthy}}{{end}} +{{range .Provision}}{{if ne .State "ready"}}{{end}}{{end}} +{{range .Warnings}}{{end}} {{end}}
NodeStateGPUsRAMDiskEnginesClassesFabricsModelsReplicasLast seen
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} {{.State}}{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}{{if .Engines}}{{join .Engines}}{{else}}none yet{{end}} {{join .Classes}} {{join .Fabrics}} {{if .Models}}{{len .Models}}{{else}}—{{end}} {{.Replicas}} {{or .LastSeenAgo "—"}}
{{join .Unhealthy}}
{{join .Unhealthy}}
{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}
{{.}}
{{else}}

No agents have registered.

{{end}}