diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go
index ecf5302..2722542 100644
--- a/api/gen/llmgw/v1/agent.pb.go
+++ b/api/gen/llmgw/v1/agent.pb.go
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9, 0}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -202,6 +202,7 @@ type ControlMessage struct {
// *ControlMessage_Drain
// *ControlMessage_Ack
// *ControlMessage_RemoveWeights
+ // *ControlMessage_Provision
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -289,6 +290,15 @@ func (x *ControlMessage) GetRemoveWeights() *RemoveWeights {
return nil
}
+func (x *ControlMessage) GetProvision() *Provision {
+ if x != nil {
+ if x, ok := x.Msg.(*ControlMessage_Provision); ok {
+ return x.Provision
+ }
+ }
+ return nil
+}
+
type isControlMessage_Msg interface {
isControlMessage_Msg()
}
@@ -313,6 +323,10 @@ type ControlMessage_RemoveWeights struct {
RemoveWeights *RemoveWeights `protobuf:"bytes,5,opt,name=remove_weights,json=removeWeights,proto3,oneof"`
}
+type ControlMessage_Provision struct {
+ Provision *Provision `protobuf:"bytes,6,opt,name=provision,proto3,oneof"`
+}
+
func (*ControlMessage_Start) isControlMessage_Msg() {}
func (*ControlMessage_Stop) isControlMessage_Msg() {}
@@ -323,6 +337,8 @@ func (*ControlMessage_Ack) isControlMessage_Msg() {}
func (*ControlMessage_RemoveWeights) isControlMessage_Msg() {}
+func (*ControlMessage_Provision) isControlMessage_Msg() {}
+
// Register is the first message on every stream. It carries the node's full
// hardware inventory, so the control plane needs no separate discovery call and
// a reconnecting agent automatically refreshes what the scheduler knows.
@@ -381,7 +397,15 @@ type Register struct {
//
// Empty means the node was not asked to look, exactly as for weights, and
// constrains nothing.
- Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"`
+ Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"`
+ // host is what the machine is, so the control plane can have the mirror
+ // fetch software built for it rather than assuming every node matches the
+ // first.
+ Host *Host `protobuf:"bytes,13,opt,name=host,proto3" json:"host,omitempty"`
+ // engines_probed says engine_versions is an answer. Without it an empty map
+ // meant "an agent too old to say" and constrained nothing — so a freshly
+ // built node with no engine installed yet was a candidate for every model.
+ EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -500,6 +524,113 @@ func (x *Register) GetStorage() []*Storage {
return nil
}
+func (x *Register) GetHost() *Host {
+ if x != nil {
+ return x.Host
+ }
+ return nil
+}
+
+func (x *Register) GetEnginesProbed() bool {
+ if x != nil {
+ return x.EnginesProbed
+ }
+ return false
+}
+
+// Host describes the operating system and processor of a node.
+type Host struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // From /etc/os-release: "ubuntu", "24.04", "noble".
+ OsId string `protobuf:"bytes,1,opt,name=os_id,json=osId,proto3" json:"os_id,omitempty"`
+ OsVersion string `protobuf:"bytes,2,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"`
+ OsCodename string `protobuf:"bytes,3,opt,name=os_codename,json=osCodename,proto3" json:"os_codename,omitempty"`
+ // Debian's spelling, which is what package indexes are keyed by: "amd64",
+ // "arm64".
+ Arch string `protobuf:"bytes,4,opt,name=arch,proto3" json:"arch,omitempty"`
+ // The vector extensions inference cares about, of those the CPU has. A VM
+ // given QEMU's default CPU model has none of them, and runs, and is five
+ // times slower, and nothing says why.
+ CpuFlags []string `protobuf:"bytes,5,rep,name=cpu_flags,json=cpuFlags,proto3" json:"cpu_flags,omitempty"`
+ // nvidia_driver is the driver version, empty when there is none. A card
+ // without a driver is not a GPU anything can use.
+ NvidiaDriver string `protobuf:"bytes,6,opt,name=nvidia_driver,json=nvidiaDriver,proto3" json:"nvidia_driver,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *Host) Reset() {
+ *x = Host{}
+ mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *Host) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Host) ProtoMessage() {}
+
+func (x *Host) ProtoReflect() protoreflect.Message {
+ mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Host.ProtoReflect.Descriptor instead.
+func (*Host) Descriptor() ([]byte, []int) {
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *Host) GetOsId() string {
+ if x != nil {
+ return x.OsId
+ }
+ return ""
+}
+
+func (x *Host) GetOsVersion() string {
+ if x != nil {
+ return x.OsVersion
+ }
+ return ""
+}
+
+func (x *Host) GetOsCodename() string {
+ if x != nil {
+ return x.OsCodename
+ }
+ return ""
+}
+
+func (x *Host) GetArch() string {
+ if x != nil {
+ return x.Arch
+ }
+ return ""
+}
+
+func (x *Host) GetCpuFlags() []string {
+ if x != nil {
+ return x.CpuFlags
+ }
+ return nil
+}
+
+func (x *Host) GetNvidiaDriver() string {
+ if x != nil {
+ return x.NvidiaDriver
+ }
+ return ""
+}
+
// Storage is one filesystem a node keeps weights on.
//
// Per filesystem rather than per configured directory: free space belongs to
@@ -522,7 +653,7 @@ type Storage struct {
func (x *Storage) Reset() {
*x = Storage{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -534,7 +665,7 @@ func (x *Storage) String() string {
func (*Storage) ProtoMessage() {}
func (x *Storage) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[4]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -547,7 +678,7 @@ func (x *Storage) ProtoReflect() protoreflect.Message {
// Deprecated: Use Storage.ProtoReflect.Descriptor instead.
func (*Storage) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4}
}
func (x *Storage) GetPath() string {
@@ -614,7 +745,7 @@ type RunningShard struct {
func (x *RunningShard) Reset() {
*x = RunningShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[4]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -626,7 +757,7 @@ func (x *RunningShard) String() string {
func (*RunningShard) ProtoMessage() {}
func (x *RunningShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[4]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[5]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -639,7 +770,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use RunningShard.ProtoReflect.Descriptor instead.
func (*RunningShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5}
}
func (x *RunningShard) GetReplicaId() string {
@@ -776,7 +907,7 @@ type GPU struct {
func (x *GPU) Reset() {
*x = GPU{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[5]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -788,7 +919,7 @@ func (x *GPU) String() string {
func (*GPU) ProtoMessage() {}
func (x *GPU) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[5]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[6]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -801,7 +932,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message {
// Deprecated: Use GPU.ProtoReflect.Descriptor instead.
func (*GPU) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
}
func (x *GPU) GetIndex() int32 {
@@ -923,7 +1054,7 @@ type FabricAttachment struct {
func (x *FabricAttachment) Reset() {
*x = FabricAttachment{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[6]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -935,7 +1066,7 @@ func (x *FabricAttachment) String() string {
func (*FabricAttachment) ProtoMessage() {}
func (x *FabricAttachment) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[6]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -948,7 +1079,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead.
func (*FabricAttachment) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
}
func (x *FabricAttachment) GetFabric() string {
@@ -990,7 +1121,7 @@ type LocalWeights struct {
func (x *LocalWeights) Reset() {
*x = LocalWeights{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[7]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1002,7 +1133,7 @@ func (x *LocalWeights) String() string {
func (*LocalWeights) ProtoMessage() {}
func (x *LocalWeights) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[7]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1015,7 +1146,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead.
func (*LocalWeights) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
}
func (x *LocalWeights) GetPath() string {
@@ -1070,13 +1201,18 @@ type Heartbeat struct {
// somebody restarts the agent — which is a step nobody knows to take.
EnginesChanged bool `protobuf:"varint,8,opt,name=engines_changed,json=enginesChanged,proto3" json:"engines_changed,omitempty"`
EngineVersions map[string]string `protobuf:"bytes,9,rep,name=engine_versions,json=engineVersions,proto3" json:"engine_versions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ // provision is where installing the node's software stands, sent when it
+ // changed. The flag makes an empty list mean "nothing to install" rather
+ // than "nothing to say", as for weights.
+ ProvisionChanged bool `protobuf:"varint,10,opt,name=provision_changed,json=provisionChanged,proto3" json:"provision_changed,omitempty"`
+ Provision []*ProvisionStatus `protobuf:"bytes,11,rep,name=provision,proto3" json:"provision,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
}
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[8]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1088,7 +1224,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[8]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1101,7 +1237,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
}
func (x *Heartbeat) GetNodeId() string {
@@ -1167,6 +1303,206 @@ func (x *Heartbeat) GetEngineVersions() map[string]string {
return nil
}
+func (x *Heartbeat) GetProvisionChanged() bool {
+ if x != nil {
+ return x.ProvisionChanged
+ }
+ return false
+}
+
+func (x *Heartbeat) GetProvision() []*ProvisionStatus {
+ if x != nil {
+ return x.Provision
+ }
+ return nil
+}
+
+// Provision is the software a node should have: which engines, at which
+// versions. Nothing more.
+//
+// Everything else is the node's to work out, because only the node knows it:
+// which build fits its hardware, where that build is published, what its
+// operating system is missing, and whether the result actually runs. The agent
+// converges on this — installs what is absent, checks what is there — and
+// reports each engine through ProvisionStatus. Nothing is taken on trust from
+// what happens to be on the machine: an engine is offered to placement only
+// once the agent has installed it and seen it run.
+type Provision struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ Engines []*EngineWant `protobuf:"bytes,1,rep,name=engines,proto3" json:"engines,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *Provision) Reset() {
+ *x = Provision{}
+ mi := &file_llmgw_v1_agent_proto_msgTypes[10]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *Provision) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Provision) ProtoMessage() {}
+
+func (x *Provision) ProtoReflect() protoreflect.Message {
+ mi := &file_llmgw_v1_agent_proto_msgTypes[10]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
+func (*Provision) Descriptor() ([]byte, []int) {
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *Provision) GetEngines() []*EngineWant {
+ if x != nil {
+ return x.Engines
+ }
+ return nil
+}
+
+// EngineWant names one engine at one version.
+type EngineWant struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"` // "llamacpp"
+ Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` // "b10991"
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *EngineWant) Reset() {
+ *x = EngineWant{}
+ mi := &file_llmgw_v1_agent_proto_msgTypes[11]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *EngineWant) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EngineWant) ProtoMessage() {}
+
+func (x *EngineWant) ProtoReflect() protoreflect.Message {
+ mi := &file_llmgw_v1_agent_proto_msgTypes[11]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
+func (*EngineWant) Descriptor() ([]byte, []int) {
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
+}
+
+func (x *EngineWant) GetEngine() string {
+ if x != nil {
+ return x.Engine
+ }
+ return ""
+}
+
+func (x *EngineWant) GetVersion() string {
+ if x != nil {
+ return x.Version
+ }
+ return ""
+}
+
+// ProvisionStatus is where one engine's installation stands on a node.
+type ProvisionStatus struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"`
+ Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
+ Variant string `protobuf:"bytes,3,opt,name=variant,proto3" json:"variant,omitempty"`
+ // "installing", "ready", "failed", or "unsupported" when no build fits.
+ State string `protobuf:"bytes,4,opt,name=state,proto3" json:"state,omitempty"`
+ // detail says what is happening or what went wrong, in words for a person.
+ Detail string `protobuf:"bytes,5,opt,name=detail,proto3" json:"detail,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *ProvisionStatus) Reset() {
+ *x = ProvisionStatus{}
+ mi := &file_llmgw_v1_agent_proto_msgTypes[12]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *ProvisionStatus) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProvisionStatus) ProtoMessage() {}
+
+func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
+ mi := &file_llmgw_v1_agent_proto_msgTypes[12]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
+func (*ProvisionStatus) Descriptor() ([]byte, []int) {
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
+}
+
+func (x *ProvisionStatus) GetEngine() string {
+ if x != nil {
+ return x.Engine
+ }
+ return ""
+}
+
+func (x *ProvisionStatus) GetVersion() string {
+ if x != nil {
+ return x.Version
+ }
+ return ""
+}
+
+func (x *ProvisionStatus) GetVariant() string {
+ if x != nil {
+ return x.Variant
+ }
+ return ""
+}
+
+func (x *ProvisionStatus) GetState() string {
+ if x != nil {
+ return x.State
+ }
+ return ""
+}
+
+func (x *ProvisionStatus) GetDetail() string {
+ if x != nil {
+ return x.Detail
+ }
+ return ""
+}
+
// ShardStatus reports a shard's lifecycle transition.
type ShardStatus struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -1193,7 +1529,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[9]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1205,7 +1541,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[9]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1218,7 +1554,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *ShardStatus) GetNodeId() string {
@@ -1292,7 +1628,7 @@ type FetchProgress struct {
func (x *FetchProgress) Reset() {
*x = FetchProgress{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[10]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1304,7 +1640,7 @@ func (x *FetchProgress) String() string {
func (*FetchProgress) ProtoMessage() {}
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[10]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1317,7 +1653,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
func (*FetchProgress) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
}
func (x *FetchProgress) GetSource() string {
@@ -1373,7 +1709,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[11]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1385,7 +1721,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[11]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[15]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1398,7 +1734,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
}
func (x *StartShard) GetReplicaId() string {
@@ -1525,7 +1861,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[12]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1537,7 +1873,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[12]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[16]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1550,7 +1886,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
}
func (x *EngineProfile) GetName() string {
@@ -1646,7 +1982,7 @@ type RemoveWeights struct {
func (x *RemoveWeights) Reset() {
*x = RemoveWeights{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[13]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[17]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1658,7 +1994,7 @@ func (x *RemoveWeights) String() string {
func (*RemoveWeights) ProtoMessage() {}
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[13]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[17]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1671,7 +2007,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
func (*RemoveWeights) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
}
func (x *RemoveWeights) GetPaths() []string {
@@ -1691,7 +2027,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[14]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[18]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1703,7 +2039,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[14]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[18]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1716,7 +2052,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
}
func (x *StopShard) GetShardId() string {
@@ -1736,7 +2072,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[15]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1748,7 +2084,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[15]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[19]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1761,7 +2097,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
}
func (x *DrainShard) GetShardId() string {
@@ -1783,7 +2119,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[16]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[20]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1795,7 +2131,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[16]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[20]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1808,7 +2144,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
}
func (x *Ack) GetMessageId() string {
@@ -1841,14 +2177,15 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
"\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shardB\x05\n" +
- "\x03msg\"\x83\x02\n" +
+ "\x03msg\"\xb8\x02\n" +
"\x0eControlMessage\x12,\n" +
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ack\x12@\n" +
- "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeightsB\x05\n" +
- "\x03msg\"\xa7\x05\n" +
+ "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeights\x123\n" +
+ "\tprovision\x18\x06 \x01(\v2\x13.llmgw.v1.ProvisionH\x00R\tprovisionB\x05\n" +
+ "\x03msg\"\xf2\x05\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -1862,13 +2199,24 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x0fmemory_total_mb\x18\n" +
" \x01(\x03R\rmemoryTotalMb\x120\n" +
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" +
- "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x1a9\n" +
+ "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" +
+ "\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" +
+ "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x1a9\n" +
"\vLabelsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
- "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"u\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb1\x01\n" +
+ "\x04Host\x12\x13\n" +
+ "\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" +
+ "\n" +
+ "os_version\x18\x02 \x01(\tR\tosVersion\x12\x1f\n" +
+ "\vos_codename\x18\x03 \x01(\tR\n" +
+ "osCodename\x12\x12\n" +
+ "\x04arch\x18\x04 \x01(\tR\x04arch\x12\x1b\n" +
+ "\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" +
+ "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\"u\n" +
"\aStorage\x12\x12\n" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
@@ -1925,7 +2273,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
- "\x06shared\x18\x03 \x01(\bR\x06shared\"\xe2\x03\n" +
+ "\x06shared\x18\x03 \x01(\bR\x06shared\"\xc8\x04\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -1935,10 +2283,25 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12&\n" +
"\x0fmemory_total_mb\x18\a \x01(\x03R\rmemoryTotalMb\x12'\n" +
"\x0fengines_changed\x18\b \x01(\bR\x0eenginesChanged\x12P\n" +
- "\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x1aA\n" +
+ "\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x12+\n" +
+ "\x11provision_changed\x18\n" +
+ " \x01(\bR\x10provisionChanged\x127\n" +
+ "\tprovision\x18\v \x03(\v2\x19.llmgw.v1.ProvisionStatusR\tprovision\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
- "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbb\x03\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +
+ "\tProvision\x12.\n" +
+ "\aengines\x18\x01 \x03(\v2\x14.llmgw.v1.EngineWantR\aengines\">\n" +
+ "\n" +
+ "EngineWant\x12\x16\n" +
+ "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" +
+ "\aversion\x18\x02 \x01(\tR\aversion\"\x8b\x01\n" +
+ "\x0fProvisionStatus\x12\x16\n" +
+ "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" +
+ "\aversion\x18\x02 \x01(\tR\aversion\x12\x18\n" +
+ "\avariant\x18\x03 \x01(\tR\avariant\x12\x14\n" +
+ "\x05state\x18\x04 \x01(\tR\x05state\x12\x16\n" +
+ "\x06detail\x18\x05 \x01(\tR\x06detail\"\xbb\x03\n" +
"\vShardStatus\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" +
"\n" +
@@ -2035,62 +2398,70 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
+var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 25)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
(*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage
(*Register)(nil), // 3: llmgw.v1.Register
- (*Storage)(nil), // 4: llmgw.v1.Storage
- (*RunningShard)(nil), // 5: llmgw.v1.RunningShard
- (*GPU)(nil), // 6: llmgw.v1.GPU
- (*FabricAttachment)(nil), // 7: llmgw.v1.FabricAttachment
- (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights
- (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat
- (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus
- (*FetchProgress)(nil), // 11: llmgw.v1.FetchProgress
- (*StartShard)(nil), // 12: llmgw.v1.StartShard
- (*EngineProfile)(nil), // 13: llmgw.v1.EngineProfile
- (*RemoveWeights)(nil), // 14: llmgw.v1.RemoveWeights
- (*StopShard)(nil), // 15: llmgw.v1.StopShard
- (*DrainShard)(nil), // 16: llmgw.v1.DrainShard
- (*Ack)(nil), // 17: llmgw.v1.Ack
- nil, // 18: llmgw.v1.Register.LabelsEntry
- nil, // 19: llmgw.v1.Register.EngineVersionsEntry
- nil, // 20: llmgw.v1.Heartbeat.EngineVersionsEntry
- nil, // 21: llmgw.v1.EngineProfile.EnvEntry
+ (*Host)(nil), // 4: llmgw.v1.Host
+ (*Storage)(nil), // 5: llmgw.v1.Storage
+ (*RunningShard)(nil), // 6: llmgw.v1.RunningShard
+ (*GPU)(nil), // 7: llmgw.v1.GPU
+ (*FabricAttachment)(nil), // 8: llmgw.v1.FabricAttachment
+ (*LocalWeights)(nil), // 9: llmgw.v1.LocalWeights
+ (*Heartbeat)(nil), // 10: llmgw.v1.Heartbeat
+ (*Provision)(nil), // 11: llmgw.v1.Provision
+ (*EngineWant)(nil), // 12: llmgw.v1.EngineWant
+ (*ProvisionStatus)(nil), // 13: llmgw.v1.ProvisionStatus
+ (*ShardStatus)(nil), // 14: llmgw.v1.ShardStatus
+ (*FetchProgress)(nil), // 15: llmgw.v1.FetchProgress
+ (*StartShard)(nil), // 16: llmgw.v1.StartShard
+ (*EngineProfile)(nil), // 17: llmgw.v1.EngineProfile
+ (*RemoveWeights)(nil), // 18: llmgw.v1.RemoveWeights
+ (*StopShard)(nil), // 19: llmgw.v1.StopShard
+ (*DrainShard)(nil), // 20: llmgw.v1.DrainShard
+ (*Ack)(nil), // 21: llmgw.v1.Ack
+ nil, // 22: llmgw.v1.Register.LabelsEntry
+ nil, // 23: llmgw.v1.Register.EngineVersionsEntry
+ nil, // 24: llmgw.v1.Heartbeat.EngineVersionsEntry
+ nil, // 25: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
- 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
- 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
- 12, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
- 15, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
- 16, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
- 17, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
- 14, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
- 18, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
- 6, // 9: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
- 7, // 10: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
- 19, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
- 5, // 12: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
- 8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
- 4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
- 0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
- 4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
- 8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
- 20, // 18: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
- 0, // 19: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
- 11, // 20: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
- 13, // 21: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
- 21, // 22: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
- 1, // 23: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
- 2, // 24: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
- 24, // [24:25] is the sub-list for method output_type
- 23, // [23:24] is the sub-list for method input_type
- 23, // [23:23] is the sub-list for extension type_name
- 23, // [23:23] is the sub-list for extension extendee
- 0, // [0:23] is the sub-list for field type_name
+ 10, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
+ 14, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
+ 16, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
+ 19, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
+ 20, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
+ 21, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
+ 18, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
+ 11, // 8: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
+ 22, // 9: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
+ 7, // 10: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
+ 8, // 11: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
+ 23, // 12: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
+ 6, // 13: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
+ 9, // 14: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
+ 5, // 15: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
+ 4, // 16: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
+ 0, // 17: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
+ 5, // 18: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
+ 9, // 19: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
+ 24, // 20: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
+ 13, // 21: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
+ 12, // 22: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
+ 0, // 23: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
+ 15, // 24: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
+ 17, // 25: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
+ 25, // 26: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
+ 1, // 27: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
+ 2, // 28: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
+ 28, // [28:29] is the sub-list for method output_type
+ 27, // [27:28] is the sub-list for method input_type
+ 27, // [27:27] is the sub-list for extension type_name
+ 27, // [27:27] is the sub-list for extension extendee
+ 0, // [0:27] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -2109,6 +2480,7 @@ func file_llmgw_v1_agent_proto_init() {
(*ControlMessage_Drain)(nil),
(*ControlMessage_Ack)(nil),
(*ControlMessage_RemoveWeights)(nil),
+ (*ControlMessage_Provision)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -2116,7 +2488,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
- NumMessages: 21,
+ NumMessages: 25,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto
index 1a54e02..79c4ffb 100644
--- a/api/proto/llmgw/v1/agent.proto
+++ b/api/proto/llmgw/v1/agent.proto
@@ -34,6 +34,7 @@ message ControlMessage {
DrainShard drain = 3;
Ack ack = 4;
RemoveWeights remove_weights = 5;
+ Provision provision = 6;
}
}
@@ -101,6 +102,34 @@ message Register {
// Empty means the node was not asked to look, exactly as for weights, and
// constrains nothing.
repeated Storage storage = 12;
+
+ // host is what the machine is, so the control plane can have the mirror
+ // fetch software built for it rather than assuming every node matches the
+ // first.
+ Host host = 13;
+
+ // engines_probed says engine_versions is an answer. Without it an empty map
+ // meant "an agent too old to say" and constrained nothing — so a freshly
+ // built node with no engine installed yet was a candidate for every model.
+ bool engines_probed = 14;
+}
+
+// Host describes the operating system and processor of a node.
+message Host {
+ // From /etc/os-release: "ubuntu", "24.04", "noble".
+ string os_id = 1;
+ string os_version = 2;
+ string os_codename = 3;
+ // Debian's spelling, which is what package indexes are keyed by: "amd64",
+ // "arm64".
+ string arch = 4;
+ // The vector extensions inference cares about, of those the CPU has. A VM
+ // given QEMU's default CPU model has none of them, and runs, and is five
+ // times slower, and nothing says why.
+ repeated string cpu_flags = 5;
+ // nvidia_driver is the driver version, empty when there is none. A card
+ // without a driver is not a GPU anything can use.
+ string nvidia_driver = 6;
}
// Storage is one filesystem a node keeps weights on.
@@ -234,6 +263,43 @@ message Heartbeat {
// somebody restarts the agent — which is a step nobody knows to take.
bool engines_changed = 8;
map engine_versions = 9;
+
+ // provision is where installing the node's software stands, sent when it
+ // changed. The flag makes an empty list mean "nothing to install" rather
+ // than "nothing to say", as for weights.
+ bool provision_changed = 10;
+ repeated ProvisionStatus provision = 11;
+}
+
+// Provision is the software a node should have: which engines, at which
+// versions. Nothing more.
+//
+// Everything else is the node's to work out, because only the node knows it:
+// which build fits its hardware, where that build is published, what its
+// operating system is missing, and whether the result actually runs. The agent
+// converges on this — installs what is absent, checks what is there — and
+// reports each engine through ProvisionStatus. Nothing is taken on trust from
+// what happens to be on the machine: an engine is offered to placement only
+// once the agent has installed it and seen it run.
+message Provision {
+ repeated EngineWant engines = 1;
+}
+
+// EngineWant names one engine at one version.
+message EngineWant {
+ string engine = 1; // "llamacpp"
+ string version = 2; // "b10991"
+}
+
+// ProvisionStatus is where one engine's installation stands on a node.
+message ProvisionStatus {
+ string engine = 1;
+ string version = 2;
+ string variant = 3;
+ // "installing", "ready", "failed", or "unsupported" when no build fits.
+ string state = 4;
+ // detail says what is happening or what went wrong, in words for a person.
+ string detail = 5;
}
// ShardStatus reports a shard's lifecycle transition.
diff --git a/deploy/bootstrap/add-node.sh b/deploy/bootstrap/add-node.sh
index 290dd89..5dd2273 100755
--- a/deploy/bootstrap/add-node.sh
+++ b/deploy/bootstrap/add-node.sh
@@ -108,6 +108,5 @@ echo "== 2/2 onboarding $NAME"
echo
echo "Check it with: llmgw ctl nodes"
echo
-echo "A node is not finished until the engines its profiles name are installed"
-echo "and the weights are on disk. Onboarding checks that a configured engine"
-echo "runs, but it installs neither — see docs/node-onboarding.md."
+echo "The agent installs its engines itself; the node takes no model until they"
+echo "are installed and seen to run. 'llmgw ctl nodes' shows where that stands."
diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh
index fdec40a..33790e8 100755
--- a/deploy/bootstrap/onboard-node.sh
+++ b/deploy/bootstrap/onboard-node.sh
@@ -33,8 +33,8 @@ usage: $0 --node ID --addr HOST --server URL [options]
address on the route to --server, found by the agent)
--mgmt-if NAME interface NCCL bootstraps over (default: the one holding
that address)
- --llamacpp-bin P path to llama-server (default: PATH, then /opt/llama.cpp)
- --vllm-bin P path to the vllm entry point
+ --llamacpp-bin P use this llama-server instead of the one the agent installs
+ --vllm-bin P path to the vllm entry point (not yet installed by the agent)
--sglang-python P python interpreter of the SGLang virtualenv
--nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
driver installs routinely put it somewhere else.
@@ -44,6 +44,8 @@ usage: $0 --node ID --addr HOST --server URL [options]
Releases carry static linux/amd64 and linux/arm64 builds;
downloading one is better than copying whatever happens to
be on the machine you are running this from.
+ --releases URL where release binaries are downloaded from (default the
+ project's Gitea releases)
--ssh-user NAME (default ilker)
USAGE
exit 2
@@ -54,6 +56,7 @@ NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
SSH_WAIT=180
PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker
+RELEASES=https://gitea.mynodes.xyz/ilker/llmgw/releases/download
FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1"
while [ $# -gt 0 ]; do
@@ -71,6 +74,7 @@ while [ $# -gt 0 ]; do
--ib-root) IB_ROOT="$2"; shift 2;;
--pki) PKI="$2"; shift 2;;
--binary) BINARY="$2"; shift 2;;
+ --releases) RELEASES="$2"; shift 2;;
--ssh-user) SSH_USER="$2"; shift 2;;
*) usage;;
esac
@@ -131,9 +135,28 @@ $SSH "set -e
rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
# 4. The binary, the configuration and the unit.
-say "installing the binary"
-scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
-$SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
+# The node fetches the release CI built for this version, for its own
+# architecture, and checks it against the release's checksums — rather than
+# being handed whatever binary is on the machine running this script. A build
+# that is not a release (a developer's, with commits past the tag) has no such
+# file, and is copied as before.
+TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
+if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
+ say "installing llmgw $TAG from the release"
+ $SSH "set -e
+ arch=\$(dpkg --print-architecture)
+ name=llmgw-$TAG-linux-\$arch
+ cd /tmp && rm -f \$name checksums.txt
+ curl -fsSL -o \$name '$RELEASES/$TAG/'\$name
+ curl -fsSL -o checksums.txt '$RELEASES/$TAG/checksums.txt'
+ grep \" \$name\\\$\" checksums.txt | sha256sum -c - >/dev/null
+ sudo install -m 0755 \$name /usr/local/bin/llmgw
+ rm -f \$name checksums.txt"
+else
+ say "installing the binary (${TAG:-unversioned}, not a release)"
+ scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
+ $SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
+fi
say "writing the configuration"
NODE_IS_HOSTNAME=no
@@ -242,8 +265,9 @@ $SSH 'set -e
echo
if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
- say "no engine paths given; this node can only run the 'sim' engine until"
- say "one is installed and the configuration points at it"
+ say "the agent now installs the engines the control plane asks for: the"
+ say "build for this machine, checked against its publisher's checksum and run"
+ say "before the node is offered any model. 'llmgw ctl nodes' shows progress."
fi
if [ -n "$ENGINE_PROBLEMS" ]; then
echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS"
diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml
index 5bd2ac5..938d388 100644
--- a/deploy/examples/agent.yaml
+++ b/deploy/examples/agent.yaml
@@ -84,6 +84,15 @@ discovery:
# - /var/lib/llmgw-models
# - /mnt/lustre/models
+# The agent installs the engines the control plane names (see `engines:` in
+# server.yaml) under /opt/llmgw/engines: the build for this machine — CUDA when
+# an NVIDIA driver is loaded — checked against its publisher's checksum, with
+# the OS packages it needs, and run before the node takes any model. llama.cpp
+# is installed this way today. Naming a binary below opts that engine out and
+# uses yours instead; vLLM and SGLang still have to be named until the agent
+# installs them too.
+# engines_dir: /opt/llmgw/engines
+
engines:
vllm:
bin: /opt/venvs/vllm/bin/vllm
diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml
index 052b291..52ddcfb 100644
--- a/deploy/examples/server.yaml
+++ b/deploy/examples/server.yaml
@@ -197,6 +197,13 @@ router:
session_load_factor: 1.25
# How aggressively the control plane reconciles.
+# The engines every node installs, and at which version. Unset keeps the
+# versions this release was checked against (llama.cpp b10991). Each agent
+# picks the build for its own hardware and installs it itself; changing a
+# version here reaches every node on its next connect.
+# engines:
+# llamacpp: b10991
+
control:
interval: 5s
base_backoff: 10s
diff --git a/docs/reference.md b/docs/reference.md
index 57dd1b5..fe3b760 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -54,6 +54,7 @@ rather than repeated.
| `adopt_ready_timeout` | `time.Duration` | [Config](#config) |
| `tls` | `tlsconf.Config` | [Config](#config) |
| `model_dirs` | `[]string` | [Config](#config) |
+| `engines_dir` | `string` | [Config](#config) |
| `weights_cache` | `string` | [Config](#config) |
| `discovery` | `discovery.Options` | [Config](#config) |
| `engines` | `engines.Config` | [Config](#config) |
@@ -160,6 +161,7 @@ rather than repeated.
| `agent_tls` | `tlsconf.Config` | [Server](#server) |
| `monitoring` | `Monitoring` | [Server](#server) |
| `mirror` | `Mirror` | [Server](#server) |
+| `engines` | `map[string]string` | [Server](#server) |
| `router` | `Router` | [Server](#server) |
| `control` | `Control` | [Server](#server) |
| `models` | `[]catalog.Model` | [Server](#server) |
@@ -432,6 +434,13 @@ exported over the fabric, local NVMe and a copy made by hand all look
identical from here, and placement only needs to know whether the file
is present.
+### `engines_dir`
+
+`string`
+
+EnginesDir is where the agent installs engines, one directory per
+build. Defaults to /opt/llmgw/engines.
+
### `weights_cache`
`string`
@@ -589,8 +598,8 @@ know that this page cannot guess.
Bin is the llama-server binary. It is the server, not the CLI: the
gateway talks OpenAI over HTTP and needs the one that serves it.
-Empty looks on PATH, then where release archives are unpacked
-(/opt/llama.cpp).
+Empty means the agent installs llama.cpp itself, at the version the
+control plane asks for. Setting it opts out of that.
### `parallel`
@@ -1193,6 +1202,18 @@ Empty means there is none, and /admin/mirror stays unmounted.
Takes a [Mirror](#mirror) block.
+### `engines`
+
+`map[string]string`
+
+Engines are the engines every node installs, and at which version. The
+agents install them themselves — the right build for their hardware,
+checked against its publisher's checksum and seen to run — so a node is
+never trusted to already have what somebody once put there.
+
+Unset keeps DefaultEngines. A version replaces the default; an empty
+one leaves that engine to be installed by hand.
+
### `router`
`Router`
diff --git a/internal/agent/agent.go b/internal/agent/agent.go
index 730738e..10def64 100644
--- a/internal/agent/agent.go
+++ b/internal/agent/agent.go
@@ -28,6 +28,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
@@ -94,6 +95,10 @@ type Config struct {
// is present.
ModelDirs []string `koanf:"model_dirs"`
+ // EnginesDir is where the agent installs engines, one directory per
+ // build. Defaults to /opt/llmgw/engines.
+ EnginesDir string `koanf:"engines_dir"`
+
// WeightsCache is where a remote source is downloaded to. Defaults to the
// first entry of ModelDirs, because a node that says where its models live
// has already answered this question, and otherwise to
@@ -304,16 +309,28 @@ type Agent struct {
reg *engines.Registry
http connectHTTPClient
- // engineOnce probes the installed engines once before registering.
+ // engineOnce probes the configured engines once before registering.
//
// Once, because probing SGLang imports it in a Python interpreter — seconds
// on a cold cache — and a reconnect is not a reason to pay that again.
- // Later changes are found by the slower re-probe the heartbeat starts, not
- // by registering again: an engine installed while the agent runs used to
- // stay invisible until somebody thought to restart the agent.
+ // Later changes are found by the slower re-probe the heartbeat starts.
engineOnce sync.Once
- // engineVers is guarded by mu once the first probe has run.
- engineVers map[string]string
+ // configuredVers are the engines the configuration names and that answer.
+ // Guarded by mu once the first probe has run.
+ configuredVers map[string]string
+ // installedVers are the engines this agent installed and saw run.
+ // Guarded by mu.
+ installedVers map[string]string
+
+ // prov installs the engines the control plane asks for. provWants is what
+ // it asked for, nil until it has; provStatus is where each stands and
+ // provSent the fingerprint of what the control plane last heard. All
+ // guarded by mu. provKick starts a pass without waiting for the timer.
+ prov *provision.Installer
+ provWants []provision.Want
+ provStatus map[string]provision.Status
+ provSent string
+ provKick chan struct{}
// enginesSent is the fingerprint of the engines the control plane last
// heard about, from registration or a heartbeat. Guarded by mu.
enginesSent string
@@ -329,8 +346,9 @@ type Agent struct {
fetcher *weights.Fetcher
// lastWeights is the fingerprint of the file list most recently sent, so a
- // heartbeat carries the list only when it changed. Touched only by the
- // heartbeat goroutine, which is the only thing that reads or writes it.
+ // heartbeat carries the list only when it changed. Guarded by mu: during a
+ // failover the old stream's heartbeat loop can still be running when the
+ // new one's starts.
lastWeights string
// fetchWatchers are the shards waiting on each source, so progress on one
@@ -363,12 +381,16 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
if err != nil {
return nil, err
}
- return &Agent{
+ a := &Agent{
cfg: cfg, log: log, sup: sup,
reg: engines.NewRegistry(cfg.Engines),
http: httpClient,
shardReplica: map[string]string{},
- }, nil
+ provStatus: map[string]provision.Status{},
+ provKick: make(chan struct{}, 1),
+ }
+ a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
+ return a, nil
}
// Supervisor exposes the process supervisor, for tests and `ctl logs`.
@@ -387,6 +409,7 @@ func (a *Agent) Run(ctx context.Context) error {
a.log.Error("hardware discovery failed", "err", err)
}
a.recoverOrphans()
+ go a.provisionLoop(ctx)
attempt := 0
next := 0
@@ -645,9 +668,13 @@ func (a *Agent) scan(ctx context.Context) error {
opts := a.cfg.Discovery
opts.ModelDirs = a.cfg.ModelDirs
inv, err := discovery.Discover(ctx, opts, a.log)
+ host := discovery.DiscoverHost()
a.mu.Lock()
a.inv = inv
+ a.prov.Host = host
a.mu.Unlock()
+ a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch,
+ "cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver)
return err
}
@@ -685,7 +712,18 @@ func (a *Agent) session(ctx context.Context, server string) error {
// engine whether or not its binary exists, so without this a node with no
// vLLM accepted a vLLM shard and failed it minutes later.
reg.EngineVersions = a.engines(parent)
+ reg.EnginesProbed = true
reg.RunningShards = a.runningShards()
+ a.mu.Lock()
+ h := a.prov.Host
+ // A new stream is a control plane that knows nothing of where installs
+ // stand, so the next heartbeat tells it again.
+ a.provSent = ""
+ a.mu.Unlock()
+ reg.Host = &llmgwv1.Host{
+ OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename,
+ Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver,
+ }
if err := stream.Send(&llmgwv1.AgentMessage{
Msg: &llmgwv1.AgentMessage_Register{Register: reg},
@@ -780,10 +818,22 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
a.reprobeEngines()
a.mu.Lock()
- if fp := enginesFingerprint(a.engineVers); a.engineVers != nil && fp != a.enginesSent {
- a.enginesSent = fp
+ if vers := a.engineReportLocked(); a.configuredVers != nil &&
+ enginesFingerprint(vers) != a.enginesSent {
+ a.enginesSent = enginesFingerprint(vers)
hb.EnginesChanged = true
- hb.EngineVersions = maps.Clone(a.engineVers)
+ hb.EngineVersions = vers
+ }
+ if fp := provisionFingerprint(a.provStatus); fp != a.provSent {
+ a.provSent = fp
+ hb.ProvisionChanged = true
+ for _, e := range slices.Sorted(maps.Keys(a.provStatus)) {
+ st := a.provStatus[e]
+ hb.Provision = append(hb.Provision, &llmgwv1.ProvisionStatus{
+ Engine: st.Engine, Version: st.Version, Variant: st.Variant,
+ State: st.State, Detail: st.Detail,
+ })
+ }
}
a.mu.Unlock()
@@ -807,8 +857,12 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
// rather than "nothing to report" — without it, the first heartbeat after
// a model was deleted would be indistinguishable from silence.
weights := discovery.ScanWeights(a.cfg.ModelDirs)
- if fp := weightsFingerprint(weights); fp != a.lastWeights {
- a.lastWeights = fp
+ fp := weightsFingerprint(weights)
+ a.mu.Lock()
+ changed := fp != a.lastWeights
+ a.lastWeights = fp
+ a.mu.Unlock()
+ if changed {
hb.WeightsChanged = true
for _, w := range weights {
hb.Weights = append(hb.Weights, &llmgwv1.LocalWeights{
@@ -899,6 +953,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
}
case *llmgwv1.ControlMessage_RemoveWeights:
a.removeWeights(m.RemoveWeights.GetPaths())
+ case *llmgwv1.ControlMessage_Provision:
+ a.setProvision(m.Provision)
case *llmgwv1.ControlMessage_Ack:
if !m.Ack.GetOk() {
a.log.Warn("control plane rejected a message",
@@ -1152,27 +1208,129 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
}
-// engines is what this node can run, probed once.
-//
-// Failures are silence rather than an error: an engine that is not installed is
-// the ordinary case on most nodes, and an agent that refused to register over
-// one would take a working node out of the fleet for a missing engine it was
-// never going to be asked to run.
+// engines is what this node can run: what the agent installed and saw run,
+// and what the configuration names and answers. Nothing merely found on the
+// machine — a llama-server on PATH is somebody's, not necessarily one that
+// works, and placing on it is how a replica fails on a node nobody watches.
func (a *Agent) engines(ctx context.Context) map[string]string {
a.engineOnce.Do(func() {
- vers := a.reg.Probe(ctx)
+ vers := a.reg.ProbeConfigured(ctx)
a.mu.Lock()
- a.engineVers, a.lastEngineProbe = vers, time.Now()
+ a.configuredVers, a.lastEngineProbe = vers, time.Now()
a.mu.Unlock()
- a.log.Info("engines available on this node",
- "engines", engines.ProbedNames(vers))
+ if len(vers) > 0 {
+ a.log.Info("engines named in the configuration",
+ "engines", engines.ProbedNames(vers))
+ }
})
a.mu.Lock()
defer a.mu.Unlock()
+ vers := a.engineReportLocked()
// Whatever registration sends is what the control plane now knows, so a
// heartbeat repeats it only once it differs.
- a.enginesSent = enginesFingerprint(a.engineVers)
- return maps.Clone(a.engineVers)
+ a.enginesSent = enginesFingerprint(vers)
+ return vers
+}
+
+// engineReportLocked merges the two sources. Installed wins: it is the one
+// that was seen to run. Called with mu held.
+func (a *Agent) engineReportLocked() map[string]string {
+ out := maps.Clone(a.configuredVers)
+ if out == nil {
+ out = map[string]string{}
+ }
+ maps.Copy(out, a.installedVers)
+ return out
+}
+
+// provisionEvery is how often installed engines are checked again. A library
+// removed by an unrelated upgrade is found here rather than by the next
+// replica to start.
+const provisionEvery = 10 * time.Minute
+
+// setProvision takes the control plane's list of wanted engines.
+func (a *Agent) setProvision(p *llmgwv1.Provision) {
+ wants := []provision.Want{}
+ for _, e := range p.GetEngines() {
+ wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion()})
+ }
+ a.mu.Lock()
+ a.provWants = wants
+ a.mu.Unlock()
+ select {
+ case a.provKick <- struct{}{}:
+ default:
+ }
+}
+
+// provisionLoop installs and re-checks the wanted engines, one pass at a time.
+func (a *Agent) provisionLoop(ctx context.Context) {
+ t := time.NewTicker(provisionEvery)
+ defer t.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-a.provKick:
+ case <-t.C:
+ }
+ a.mu.Lock()
+ wants := slices.Clone(a.provWants)
+ managed := a.provWants != nil
+ a.mu.Unlock()
+ if !managed {
+ continue
+ }
+
+ statuses := a.prov.Converge(ctx, wants)
+ installed := map[string]string{}
+ keep := map[string]bool{}
+ for _, st := range statuses {
+ keep[st.Engine] = true
+ if st.State == provision.Ready {
+ a.reg.UseInstalled(st.Engine, st.Binary)
+ installed[st.Engine] = st.Version
+ }
+ }
+ a.mu.Lock()
+ if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
+ a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
+ }
+ a.installedVers = installed
+ // An engine no longer asked for is no longer reported on.
+ for e := range a.provStatus {
+ if !keep[e] {
+ delete(a.provStatus, e)
+ }
+ }
+ a.mu.Unlock()
+ }
+}
+
+// provisionStatus records a change in one engine's installation.
+func (a *Agent) provisionStatus(st provision.Status) {
+ a.mu.Lock()
+ prev, had := a.provStatus[st.Engine]
+ a.provStatus[st.Engine] = st
+ a.mu.Unlock()
+ if !had || prev.State != st.State || prev.Detail != st.Detail {
+ lvl := slog.LevelInfo
+ if st.State == provision.Failed || st.State == provision.Unsupported {
+ lvl = slog.LevelWarn
+ }
+ a.log.Log(context.Background(), lvl, "engine installation",
+ "engine", st.Engine, "version", st.Version, "variant", st.Variant,
+ "state", st.State, "detail", st.Detail)
+ }
+}
+
+func provisionFingerprint(m map[string]provision.Status) string {
+ var b strings.Builder
+ for _, k := range slices.Sorted(maps.Keys(m)) {
+ st := m[k]
+ fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
+ }
+ return b.String()
}
// engineReprobe is how often the installed engines are looked for again.
@@ -1198,14 +1356,14 @@ func (a *Agent) reprobeEngines() {
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
- vers := a.reg.Probe(ctx)
+ vers := a.reg.ProbeConfigured(ctx)
a.mu.Lock()
- changed := enginesFingerprint(vers) != enginesFingerprint(a.engineVers)
- a.engineVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
+ changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
+ a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
a.mu.Unlock()
if changed {
- a.log.Info("engines available on this node changed",
+ a.log.Info("engines named in the configuration changed",
"engines", engines.ProbedNames(vers))
}
}()
diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go
new file mode 100644
index 0000000..f0dbccc
--- /dev/null
+++ b/internal/agent/discovery/host.go
@@ -0,0 +1,135 @@
+package discovery
+
+import (
+ "bufio"
+ "os"
+ "runtime"
+ "strings"
+)
+
+// Host is what a node is, as far as installing software on it is concerned.
+type Host struct {
+ // OSID, OSVersion and OSCodename come from /etc/os-release: "ubuntu",
+ // "24.04", "noble".
+ OSID string `json:"os_id,omitempty"`
+ OSVersion string `json:"os_version,omitempty"`
+ OSCodename string `json:"os_codename,omitempty"`
+ // Arch is Debian's spelling, "amd64" or "arm64", because that is what
+ // package names and release archives are keyed by.
+ Arch string `json:"arch,omitempty"`
+ // CPUFlags are the vector extensions inference cares about, of those this
+ // processor has.
+ CPUFlags []string `json:"cpu_flags,omitempty"`
+ // NVIDIADriver is the loaded driver's version, empty when there is none.
+ NVIDIADriver string `json:"nvidia_driver,omitempty"`
+}
+
+// HasCPUFlag reports whether the processor has an extension.
+func (h Host) HasCPUFlag(f string) bool {
+ for _, have := range h.CPUFlags {
+ if have == f {
+ return true
+ }
+ }
+ return false
+}
+
+// interestingFlags are the extensions that decide which build of an engine
+// runs well. A VM on QEMU's default CPU model has none of them — and still
+// runs, five times slower, with nothing saying why.
+var interestingFlags = []string{"avx", "avx2", "fma", "f16c", "avx512f", "avx512_bf16", "amx_tile", "asimd", "sve"}
+
+// DiscoverHost reads what the machine is.
+func DiscoverHost() Host {
+ h := hostFromOSRelease("/etc/os-release")
+ h.Arch = debianArch(runtime.GOARCH)
+ h.CPUFlags = cpuFlagsFrom("/proc/cpuinfo")
+ h.NVIDIADriver = nvidiaDriverFrom("/proc/driver/nvidia/version")
+ return h
+}
+
+func hostFromOSRelease(path string) Host {
+ var h Host
+ f, err := os.Open(path) //nolint:gosec // a fixed system path
+ if err != nil {
+ return h
+ }
+ defer f.Close()
+ sc := bufio.NewScanner(f)
+ for sc.Scan() {
+ k, v, ok := strings.Cut(sc.Text(), "=")
+ if !ok {
+ continue
+ }
+ v = strings.Trim(v, `"'`)
+ switch k {
+ case "ID":
+ h.OSID = v
+ case "VERSION_ID":
+ h.OSVersion = v
+ case "VERSION_CODENAME":
+ h.OSCodename = v
+ }
+ }
+ return h
+}
+
+func debianArch(goarch string) string {
+ switch goarch {
+ case "amd64", "arm64":
+ return goarch
+ case "386":
+ return "i386"
+ case "arm":
+ return "armhf"
+ }
+ return goarch
+}
+
+func cpuFlagsFrom(path string) []string {
+ b, err := os.ReadFile(path) //nolint:gosec // a fixed system path
+ if err != nil {
+ return nil
+ }
+ have := map[string]bool{}
+ for _, line := range strings.Split(string(b), "\n") {
+ k, v, ok := strings.Cut(line, ":")
+ if !ok {
+ continue
+ }
+ // "flags" on x86, "Features" on ARM. The first processor is enough:
+ // a machine does not mix instruction sets across its cores.
+ if k = strings.TrimSpace(k); k != "flags" && k != "Features" {
+ continue
+ }
+ for _, f := range strings.Fields(v) {
+ have[f] = true
+ }
+ break
+ }
+ var out []string
+ for _, f := range interestingFlags {
+ if have[f] {
+ out = append(out, f)
+ }
+ }
+ return out
+}
+
+// nvidiaDriverFrom reads the loaded kernel module's version, which exists
+// exactly when a driver is loaded — unlike nvidia-smi, which can be installed
+// on a machine whose module failed to load.
+func nvidiaDriverFrom(path string) string {
+ b, err := os.ReadFile(path) //nolint:gosec // a fixed system path
+ if err != nil {
+ return ""
+ }
+ // "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ..."
+ fields := strings.Fields(string(b))
+ for i, f := range fields {
+ if f == "Module" && i+1 < len(fields) {
+ return fields[i+1]
+ }
+ }
+ return ""
+}
diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go
index 2712be8..145d948 100644
--- a/internal/agent/engines/engine.go
+++ b/internal/agent/engines/engine.go
@@ -151,6 +151,16 @@ func NewRegistry(cfg Config) *Registry {
return r
}
+// UseInstalled points an engine at a build the agent installed, and reports
+// whether the engine takes one.
+func (r *Registry) UseInstalled(name, path string) bool {
+ u, ok := r.engines[name].(interface{ UseInstalled(string) })
+ if ok {
+ u.UseInstalled(path)
+ }
+ return ok
+}
+
// Add registers an engine, replacing any engine of the same name.
func (r *Registry) Add(e Engine) { r.engines[e.Name()] = e }
diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go
index 56b2b1b..26f7676 100644
--- a/internal/agent/engines/llamacpp.go
+++ b/internal/agent/engines/llamacpp.go
@@ -3,19 +3,19 @@ package engines
import (
"fmt"
"os"
- "os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
+ "sync/atomic"
)
// LlamaCppConfig locates the llama.cpp installation.
type LlamaCppConfig struct {
// Bin is the llama-server binary. It is the server, not the CLI: the
// gateway talks OpenAI over HTTP and needs the one that serves it.
- // Empty looks on PATH, then where release archives are unpacked
- // (/opt/llama.cpp).
+ // Empty means the agent installs llama.cpp itself, at the version the
+ // control plane asks for. Setting it opts out of that.
Bin string `koanf:"bin"`
// Parallel is how many requests a server handles at once, when a profile
// does not say. llama.cpp defaults to one, which would make every second
@@ -29,7 +29,10 @@ type LlamaCppConfig struct {
// small model on CPU at a speed that is fine for testing the whole path, and it
// serves the same OpenAI API as the others, so nothing downstream can tell the
// difference.
-type LlamaCpp struct{ cfg LlamaCppConfig }
+type LlamaCpp struct {
+ cfg LlamaCppConfig
+ installed atomic.Pointer[string]
+}
// Name identifies the engine in the catalog.
func (*LlamaCpp) Name() string { return "llamacpp" }
@@ -170,33 +173,25 @@ func ggufIn(path string) (string, error) {
return filepath.Join(path, names[0]), nil
}
-// llamaServerPlaces are where llama.cpp's release archives are unpacked when
-// they are not on PATH. The archive carries its shared libraries beside the
-// binary, so it is usually left in one directory rather than installed.
-var llamaServerPlaces = []string{
- "/opt/llama.cpp/llama-server",
- "/opt/llama.cpp/bin/llama-server",
- "/opt/llama.cpp/build/bin/llama-server",
- "/usr/local/llama.cpp/llama-server",
-}
+// UseInstalled points the engine at a build the agent installed and checked.
+// It takes precedence over everything else, because it is the only one that
+// was seen to run.
+func (l *LlamaCpp) UseInstalled(path string) { l.installed.Store(&path) }
-// bin is the llama-server to run: the configured one, else the one on PATH,
-// else the first of the usual unpack locations that exists.
-//
-// Found rather than required in the configuration, because every node here
-// had the same path written into its file by hand, and a node set up without
-// it reported no engine at all.
+// Configured reports whether the agent's configuration names a binary. A
+// named binary is the operator's deliberate choice; one merely found on PATH
+// is not, and is not reported as installed.
+func (l *LlamaCpp) Configured() bool { return l.cfg.Bin != "" }
+
+// bin is the llama-server to run: the installed build, else the configured
+// one, else whatever PATH has — which is only ever reached by a node not
+// provisioned by the agent and not told where its engine is.
func (l *LlamaCpp) bin() string {
+ if p := l.installed.Load(); p != nil {
+ return *p
+ }
if l.cfg.Bin != "" {
return l.cfg.Bin
}
- if p, err := exec.LookPath("llama-server"); err == nil {
- return p
- }
- for _, p := range llamaServerPlaces {
- if info, err := os.Stat(p); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
- return p
- }
- }
return "llama-server"
}
diff --git a/internal/agent/engines/probe.go b/internal/agent/engines/probe.go
index 51edd80..15a0ebb 100644
--- a/internal/agent/engines/probe.go
+++ b/internal/agent/engines/probe.go
@@ -157,11 +157,49 @@ func firstVersion(s string) string {
return Unknown
}
+// Configured is an engine whose location the agent's configuration names.
+type Configured interface {
+ Configured() bool
+}
+
+// ProbeConfigured probes only the engines the configuration names.
+//
+// Probe looks everywhere — PATH, the default interpreter — and reports what
+// it finds, which is exactly the trust in whatever happens to be on a machine
+// that installing engines replaces. A node the agent provisions reports what
+// it installed, plus what the operator named on purpose; nothing else.
+func (r *Registry) ProbeConfigured(ctx context.Context) map[string]string {
+ out := map[string]string{}
+ for name, e := range r.engines {
+ c, ok := e.(Configured)
+ if !ok || !c.Configured() {
+ continue
+ }
+ p, ok := e.(Prober)
+ if !ok {
+ continue
+ }
+ if v, err := p.Probe(ctx); err == nil {
+ if v == "" {
+ v = Unknown
+ }
+ out[name] = v
+ }
+ }
+ return out
+}
+
// Probe reports whether the llama.cpp server is installed.
func (l *LlamaCpp) Probe(context.Context) (string, error) {
return installedBinary(l.bin())
}
+// Configured reports whether the configuration names vLLM.
+func (v *VLLM) Configured() bool { return v.cfg.Bin != "" }
+
+// Configured reports whether the configuration names SGLang's interpreter.
+func (g *SGLang) Configured() bool { return g.cfg.Python != "" }
+
// Probe reports whether vLLM is installed.
func (v *VLLM) Probe(context.Context) (string, error) {
bin := v.cfg.Bin
diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go
new file mode 100644
index 0000000..2c11357
--- /dev/null
+++ b/internal/agent/provision/install.go
@@ -0,0 +1,534 @@
+package provision
+
+import (
+ "archive/tar"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "log/slog"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+// Want is one engine at one version.
+type Want struct {
+ Engine string
+ Version string
+}
+
+// The states an engine's installation can be in.
+const (
+ Installing = "installing"
+ Ready = "ready"
+ Failed = "failed"
+ Unsupported = "unsupported"
+)
+
+// Status is where one engine's installation stands.
+type Status struct {
+ Engine string
+ Version string
+ Variant string
+ State string
+ // Detail is for a person: what is happening, or what went wrong.
+ Detail string
+ // Binary is the executable, once ready.
+ Binary string
+}
+
+// Installer installs engines under one directory.
+type Installer struct {
+ // Root is where engines are installed, one directory per build:
+ // //-.
+ Root string
+ Host discovery.Host
+ Log *slog.Logger
+ HTTP *http.Client
+ // OnStatus hears every change, so a download that takes minutes is seen
+ // to be under way rather than silent.
+ OnStatus func(Status)
+
+ // Run executes a command; a seam for tests. Nil runs it for real.
+ Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
+ // APIBase and DownloadBase default to GitHub; tests point them at a fake.
+ APIBase string
+ DownloadBase string
+}
+
+// DefaultRoot is where engines are installed when nothing says otherwise.
+const DefaultRoot = "/opt/llmgw/engines"
+
+// marker is written into a build's directory once it has been installed and
+// seen to run. Its absence means the directory is not to be trusted.
+const marker = ".llmgw-installed"
+
+type markerFile struct {
+ Engine string `json:"engine"`
+ Version string `json:"version"`
+ Variant string `json:"variant"`
+ Binary string `json:"binary"`
+ SHA256 map[string]string `json:"sha256"`
+ Installed time.Time `json:"installed"`
+}
+
+// Converge brings every wanted engine to ready, or says why it cannot.
+//
+// An engine already installed is checked again rather than believed: its
+// directory can be damaged, or a library it needs removed by somebody's
+// upgrade, and that is found here rather than by the next replica to fail.
+func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
+ out := make([]Status, 0, len(wants))
+ for _, w := range wants {
+ st := in.one(ctx, w)
+ in.report(st)
+ out = append(out, st)
+ }
+ return out
+}
+
+func (in *Installer) one(ctx context.Context, w Want) Status {
+ st := Status{Engine: w.Engine, Version: w.Version}
+ recipe, ok := Recipes[w.Engine]
+ if !ok {
+ st.State = Unsupported
+ st.Detail = "this agent does not know how to install " + w.Engine +
+ "; install it by hand and name it in the agent's configuration"
+ return st
+ }
+ build, err := recipe.Choose(w.Version, in.Host)
+ if err != nil {
+ st.State, st.Detail = Unsupported, err.Error()
+ return st
+ }
+ st.Variant = build.Variant
+ dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
+
+ // Installed before: check it still runs, and trust nothing else about it.
+ if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
+ // The packages are checked too. Removing one is exactly the kind of
+ // change an unrelated upgrade makes, and ldd below would catch it only
+ // as a missing library with no hint of which package held it.
+ if err := in.packages(ctx, build.Packages, &st); err != nil {
+ st.State, st.Detail = Failed, err.Error()
+ return st
+ }
+ bin, version, err := in.verify(ctx, dir, build)
+ if err == nil {
+ return in.ready(st, build, bin, version)
+ }
+ in.log().Warn("an installed engine no longer runs; installing it again",
+ "engine", w.Engine, "version", w.Version, "err", err)
+ }
+
+ if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
+ st.State, st.Detail, st.Binary = Failed, err.Error(), ""
+ return st
+ }
+ bin, version, err := in.verify(ctx, dir, build)
+ if err != nil {
+ st.State, st.Detail = Failed, err.Error()
+ return st
+ }
+ return in.ready(st, build, bin, version)
+}
+
+func (in *Installer) ready(st Status, b Build, bin, version string) Status {
+ st.State, st.Binary = Ready, bin
+ st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
+ var missing []string
+ for _, f := range b.CPUFlags {
+ if !in.Host.HasCPUFlag(f) {
+ missing = append(missing, f)
+ }
+ }
+ if len(missing) > 0 {
+ // Said on every report, not once in a log: it is the difference between
+ // five tokens a second and forty, and a VM given QEMU's default CPU
+ // model looks entirely healthy otherwise.
+ st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
+ ", so it will run several times slower (a VM needs CPU type \"host\")"
+ }
+ return st
+}
+
+// install fetches, checks and unpacks a build into dir.
+//
+// Into a scratch directory renamed into place at the end, so a failure halfway
+// leaves nothing that looks installed.
+func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
+ if err := in.packages(ctx, b.Packages, st); err != nil {
+ return err
+ }
+
+ in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
+ sums, err := in.checksums(ctx, r.Repo, w.Version)
+ if err != nil {
+ return err
+ }
+
+ tmp := dir + ".partial"
+ _ = os.RemoveAll(tmp)
+ if err := os.MkdirAll(tmp, 0o755); err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
+
+ got := map[string]string{}
+ for _, asset := range b.Assets {
+ want, ok := sums[asset]
+ if !ok {
+ // Refused rather than installed unchecked: this is a binary that
+ // will run as root on every node.
+ return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
+ w.Engine, w.Version, asset)
+ }
+ archive := filepath.Join(tmp, asset+".download")
+ if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
+ return err
+ }
+ in.progress(st, "unpacking %s", asset)
+ if err := untarGz(archive, tmp); err != nil {
+ return fmt.Errorf("unpacking %s: %w", asset, err)
+ }
+ _ = os.Remove(archive)
+ got[asset] = want
+ }
+
+ in.progress(st, "checking that %s runs", b.Binary)
+ bin, _, err := in.verify(ctx, tmp, b)
+ if err != nil {
+ return err
+ }
+ rel, _ := filepath.Rel(tmp, bin)
+ m, _ := json.MarshalIndent(markerFile{
+ Engine: w.Engine, Version: w.Version, Variant: b.Variant,
+ Binary: rel, SHA256: got, Installed: time.Now().UTC(),
+ }, "", " ")
+ if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
+ return err
+ }
+ _ = os.RemoveAll(dir)
+ if err := os.Rename(tmp, dir); err != nil {
+ return err
+ }
+ in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
+ "variant", b.Variant, "dir", dir)
+ return nil
+}
+
+// verify finds the binary and runs it.
+//
+// Two checks, because they fail differently. ldd names a missing library
+// outright, where running the binary would only say it could not start. And
+// running it is the only proof that the rest — the right instruction set, the
+// right driver — is in place.
+func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
+ bin, err = findFile(dir, b.Binary)
+ if err != nil {
+ return "", "", err
+ }
+ if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
+ var missing []string
+ for _, line := range strings.Split(string(out), "\n") {
+ if strings.Contains(line, "not found") {
+ missing = append(missing, strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]))
+ }
+ }
+ if len(missing) > 0 {
+ return "", "", fmt.Errorf("%s cannot load %s", b.Binary, strings.Join(missing, ", "))
+ }
+ }
+ tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+ defer cancel()
+ out, err := in.run(tctx, nil, bin, "--version")
+ if err != nil {
+ return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
+ }
+ return bin, firstVersionLine(out), nil
+}
+
+// packages installs the operating system packages a build needs.
+//
+// Only those missing, and from the node's own package sources: a package the
+// base image already has is not touched, so an engine never upgrades
+// something the rest of the machine depends on.
+func (in *Installer) packages(ctx context.Context, pkgs []string, st *Status) error {
+ var missing []string
+ for _, p := range pkgs {
+ out, err := in.run(ctx, nil, "dpkg-query", "-W", "-f=${db:Status-Abbrev}", p)
+ if err != nil || !strings.HasPrefix(string(out), "ii") {
+ missing = append(missing, p)
+ }
+ }
+ if len(missing) == 0 {
+ return nil
+ }
+ in.progress(st, "installing %s", strings.Join(missing, ", "))
+ env := []string{"DEBIAN_FRONTEND=noninteractive"}
+ // A lock timeout rather than failing on the lock: a machine a minute old is
+ // running cloud-init's and unattended-upgrades' own apt, and the first
+ // attempt on every new node would otherwise fail.
+ lock := "-o=DPkg::Lock::Timeout=600"
+ if out, err := in.run(ctx, env, "apt-get", lock, "update"); err != nil {
+ return fmt.Errorf("apt-get update: %v: %s", err, lastLine(out))
+ }
+ args := append([]string{lock, "install", "-y", "--no-install-recommends"}, missing...)
+ if out, err := in.run(ctx, env, "apt-get", args...); err != nil {
+ return fmt.Errorf("installing %s: %v: %s", strings.Join(missing, ", "), err, lastLine(out))
+ }
+ return nil
+}
+
+// checksums reads the sha256 GitHub publishes for each asset of a release.
+func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
+ url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Accept", "application/vnd.github+json")
+ resp, err := in.client().Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
+ }
+ defer resp.Body.Close()
+ switch {
+ case resp.StatusCode == http.StatusNotFound:
+ return nil, fmt.Errorf("%s has no release %s", repo, tag)
+ case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
+ // Unauthenticated callers get sixty requests an hour per address, and
+ // a whole fleet behind one NAT shares them.
+ return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
+ case resp.StatusCode != http.StatusOK:
+ return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
+ }
+ var rel struct {
+ Assets []struct {
+ Name string `json:"name"`
+ Digest string `json:"digest"`
+ } `json:"assets"`
+ }
+ if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
+ return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
+ }
+ out := map[string]string{}
+ for _, a := range rel.Assets {
+ if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
+ out[a.Name] = sum
+ }
+ }
+ return out, nil
+}
+
+func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
+ url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return err
+ }
+ resp, err := in.client().Do(req)
+ if err != nil {
+ return fmt.Errorf("downloading %s: %w", asset, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("downloading %s: %s", asset, resp.Status)
+ }
+ if resp.ContentLength > 0 {
+ in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
+ } else {
+ in.progress(st, "downloading %s", asset)
+ }
+ f, err := os.Create(dst) //nolint:gosec // inside the install root
+ if err != nil {
+ return err
+ }
+ h := sha256.New()
+ _, err = io.Copy(io.MultiWriter(f, h), resp.Body)
+ if cerr := f.Close(); err == nil {
+ err = cerr
+ }
+ if err != nil {
+ return fmt.Errorf("downloading %s: %w", asset, err)
+ }
+ if got := hex.EncodeToString(h.Sum(nil)); got != want {
+ return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
+ asset, got[:12], want[:12])
+ }
+ return nil
+}
+
+// untarGz unpacks an archive under dir, refusing anything that would land
+// outside it. A release archive is somebody else's file, and a path of
+// "../../etc/cron.d/x" in it would otherwise be written as root.
+func untarGz(archive, dir string) error {
+ f, err := os.Open(archive) //nolint:gosec // inside the install root
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ gz, err := gzip.NewReader(f)
+ if err != nil {
+ return err
+ }
+ tr := tar.NewReader(gz)
+ for {
+ h, err := tr.Next()
+ if errors.Is(err, io.EOF) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ target, err := within(dir, h.Name)
+ if err != nil {
+ return err
+ }
+ switch h.Typeflag {
+ case tar.TypeDir:
+ if err := os.MkdirAll(target, 0o755); err != nil {
+ return err
+ }
+ case tar.TypeReg:
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return err
+ }
+ out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
+ if err != nil {
+ return err
+ }
+ if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
+ out.Close()
+ return err
+ }
+ if err := out.Close(); err != nil {
+ return err
+ }
+ case tar.TypeSymlink:
+ // Libraries are versioned through symlinks, and those are
+ // relative. One pointing anywhere else is refused.
+ if filepath.IsAbs(h.Linkname) {
+ return fmt.Errorf("%s links outside the archive", h.Name)
+ }
+ if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
+ return fmt.Errorf("%s links outside the archive", h.Name)
+ }
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return err
+ }
+ _ = os.Remove(target)
+ if err := os.Symlink(h.Linkname, target); err != nil {
+ return err
+ }
+ default:
+ // Devices, FIFOs and hard links have no business in a release.
+ }
+ }
+}
+
+func within(dir, name string) (string, error) {
+ target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
+ rel, err := filepath.Rel(dir, target)
+ if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+ return "", fmt.Errorf("%s would unpack outside the install directory", name)
+ }
+ return target, nil
+}
+
+func findFile(dir, name string) (string, error) {
+ var found string
+ _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+ if err != nil || found != "" {
+ return nil //nolint:nilerr // an unreadable subtree is not the binary
+ }
+ if !d.IsDir() && d.Name() == name {
+ found = p
+ }
+ return nil
+ })
+ if found == "" {
+ return "", fmt.Errorf("the build has no %s in it", name)
+ }
+ return found, nil
+}
+
+func (in *Installer) progress(st *Status, format string, args ...any) {
+ st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
+ in.report(*st)
+}
+
+func (in *Installer) report(st Status) {
+ if in.OnStatus != nil {
+ in.OnStatus(st)
+ }
+}
+
+func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
+ if in.Run != nil {
+ return in.Run(ctx, env, name, args...)
+ }
+ cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
+ cmd.Env = append(os.Environ(), env...)
+ return cmd.CombinedOutput()
+}
+
+func (in *Installer) root() string {
+ if in.Root != "" {
+ return in.Root
+ }
+ return DefaultRoot
+}
+
+func (in *Installer) log() *slog.Logger {
+ if in.Log != nil {
+ return in.Log
+ }
+ return slog.Default()
+}
+
+func (in *Installer) client() *http.Client {
+ if in.HTTP != nil {
+ return in.HTTP
+ }
+ return &http.Client{Timeout: 30 * time.Minute}
+}
+
+func (in *Installer) apiBase() string {
+ if in.APIBase != "" {
+ return in.APIBase
+ }
+ return "https://api.github.com"
+}
+
+func (in *Installer) downloadBase() string {
+ if in.DownloadBase != "" {
+ return in.DownloadBase
+ }
+ return "https://github.com"
+}
+
+func lastLine(b []byte) string {
+ lines := strings.Split(strings.TrimSpace(string(b)), "\n")
+ return strings.TrimSpace(lines[len(lines)-1])
+}
+
+func firstVersionLine(b []byte) string {
+ for _, l := range strings.Split(string(b), "\n") {
+ if strings.Contains(strings.ToLower(l), "version") {
+ return strings.TrimSpace(l)
+ }
+ }
+ return ""
+}
diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go
new file mode 100644
index 0000000..70808f4
--- /dev/null
+++ b/internal/agent/provision/install_test.go
@@ -0,0 +1,254 @@
+package provision
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+var ubuntu = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64", CPUFlags: []string{"avx2"}}
+
+// release is a fake GitHub: one release, its assets and their digests.
+type release struct {
+ assets map[string][]byte
+ digests map[string]string
+ gets int
+ mu sync.Mutex
+}
+
+func tarGz(t *testing.T, files map[string]string, links map[string]string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(gz)
+ for name, body := range files {
+ _ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), Typeflag: tar.TypeReg})
+ _, _ = tw.Write([]byte(body))
+ }
+ for name, target := range links {
+ _ = tw.WriteHeader(&tar.Header{Name: name, Linkname: target, Typeflag: tar.TypeSymlink})
+ }
+ _ = tw.Close()
+ _ = gz.Close()
+ return buf.Bytes()
+}
+
+func (r *release) serve(t *testing.T) *httptest.Server {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ r.mu.Lock()
+ r.gets++
+ r.mu.Unlock()
+ if strings.HasPrefix(req.URL.Path, "/repos/") {
+ var out struct {
+ Assets []map[string]string `json:"assets"`
+ }
+ for name := range r.assets {
+ out.Assets = append(out.Assets, map[string]string{"name": name, "digest": r.digests[name]})
+ }
+ _ = json.NewEncoder(w).Encode(out)
+ return
+ }
+ body, ok := r.assets[filepath.Base(req.URL.Path)]
+ if !ok {
+ http.NotFound(w, req)
+ return
+ }
+ _, _ = w.Write(body)
+ }))
+ t.Cleanup(srv.Close)
+ return srv
+}
+
+func newRelease(assets map[string][]byte) *release {
+ r := &release{assets: assets, digests: map[string]string{}}
+ for name, b := range assets {
+ sum := sha256.Sum256(b)
+ r.digests[name] = "sha256:" + hex.EncodeToString(sum[:])
+ }
+ return r
+}
+
+// fakeSystem answers dpkg-query, apt-get, ldd and the engine itself.
+type fakeSystem struct {
+ mu sync.Mutex
+ installed map[string]bool
+ aptRuns int
+ lddOut string
+ calls []string
+}
+
+func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...string) ([]byte, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.calls = append(f.calls, name+" "+strings.Join(args, " "))
+ switch {
+ case name == "dpkg-query":
+ if f.installed[args[len(args)-1]] {
+ return []byte("ii "), nil
+ }
+ return []byte("un "), errors.New("exit 1")
+ case name == "apt-get":
+ f.aptRuns++
+ if args[1] == "install" {
+ for _, p := range args[4:] {
+ f.installed[p] = true
+ }
+ }
+ return nil, nil
+ case name == "ldd":
+ return []byte(f.lddOut), nil
+ case strings.HasSuffix(name, "llama-server"):
+ return []byte("version: 10991 (abc)\n"), nil
+ }
+ return nil, errors.New("unexpected command " + name)
+}
+
+func installer(t *testing.T, srv *httptest.Server, sys *fakeSystem, host discovery.Host) (*Installer, *[]Status) {
+ var seen []Status
+ var mu sync.Mutex
+ return &Installer{
+ Root: t.TempDir(), Host: host, Run: sys.run,
+ APIBase: srv.URL, DownloadBase: srv.URL, HTTP: srv.Client(),
+ OnStatus: func(s Status) { mu.Lock(); seen = append(seen, s); mu.Unlock() },
+ }, &seen
+}
+
+func llamaRelease(t *testing.T) *release {
+ return newRelease(map[string][]byte{
+ "llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t,
+ map[string]string{"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libllama.so.0": "x"},
+ map[string]string{"llama-b1/libllama.so": "libllama.so.0"}),
+ })
+}
+
+func TestAnEngineIsInstalledFromNothing(t *testing.T) {
+ // A node rebuilt from a bare image: no engine, no OpenMP runtime. What it
+ // ends with has to have come from the publisher, checked, and been run.
+ rel := llamaRelease(t)
+ sys := &fakeSystem{installed: map[string]bool{}}
+ in, seen := installer(t, rel.serve(t), sys, ubuntu)
+
+ got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+ if len(got) != 1 || got[0].State != Ready {
+ t.Fatalf("not ready: %+v", got)
+ }
+ if !sys.installed["libgomp1"] {
+ t.Error("the OpenMP runtime the build links against was not installed")
+ }
+ if !strings.HasPrefix(got[0].Binary, in.Root) {
+ t.Errorf("binary %s is not the one installed under %s", got[0].Binary, in.Root)
+ }
+ if _, err := os.Lstat(filepath.Join(filepath.Dir(got[0].Binary), "libllama.so")); err != nil {
+ t.Errorf("the library symlink was not unpacked: %v", err)
+ }
+ var states []string
+ for _, s := range *seen {
+ states = append(states, s.State+": "+s.Detail)
+ }
+ if !strings.Contains(strings.Join(states, "\n"), "downloading") {
+ t.Errorf("a download was not reported while it ran:\n%s", strings.Join(states, "\n"))
+ }
+}
+
+func TestAnInstalledEngineIsCheckedNotFetchedAgain(t *testing.T) {
+ rel := llamaRelease(t)
+ sys := &fakeSystem{installed: map[string]bool{}}
+ in, _ := installer(t, rel.serve(t), sys, ubuntu)
+ w := []Want{{Engine: "llamacpp", Version: "b1"}}
+ in.Converge(t.Context(), w)
+ before := rel.gets
+
+ got := in.Converge(t.Context(), w)
+ if got[0].State != Ready {
+ t.Fatalf("second pass: %+v", got[0])
+ }
+ if rel.gets != before {
+ t.Errorf("a ready engine was downloaded again (%d requests)", rel.gets-before)
+ }
+}
+
+func TestAnEngineThatCannotLoadALibraryIsNotReady(t *testing.T) {
+ // Found on a node before: llama-server present, executable, and unable to
+ // start for want of a library — while the node reported ready.
+ rel := llamaRelease(t)
+ sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
+ lddOut: "\tlibcuda.so.1 => not found\n"}
+ in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+ got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+ if got[0].State != Failed || !strings.Contains(got[0].Detail, "libcuda.so.1") {
+ t.Fatalf("expected a failure naming the library: %+v", got[0])
+ }
+ if _, err := os.Stat(filepath.Join(in.Root, "llamacpp", "b1-cpu", marker)); err == nil {
+ t.Error("a build that does not run was marked installed")
+ }
+}
+
+func TestADownloadThatDoesNotMatchItsChecksumIsRefused(t *testing.T) {
+ rel := llamaRelease(t)
+ for name := range rel.digests {
+ rel.digests[name] = "sha256:" + strings.Repeat("0", 64)
+ }
+ sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
+ in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+ got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+ if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") {
+ t.Fatalf("a tampered download was not refused: %+v", got[0])
+ }
+}
+
+func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) {
+ rel := newRelease(map[string][]byte{
+ "llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t,
+ map[string]string{"../../escaped": "x", "llama-server": ""}, nil),
+ })
+ sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
+ in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+ got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+ if got[0].State != Failed {
+ t.Fatalf("an archive escaping its directory was installed: %+v", got[0])
+ }
+}
+
+func TestTheBuildFollowsTheHardware(t *testing.T) {
+ gpu := ubuntu
+ gpu.NVIDIADriver = "570.86.15"
+ b, err := chooseLlamaCpp("b1", gpu)
+ if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 {
+ t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err)
+ }
+ if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "rocky", Arch: "amd64"}); err == nil {
+ t.Error("a build was chosen for a system llama.cpp does not publish for")
+ }
+
+ slow := ubuntu
+ slow.CPUFlags = nil
+ st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
+ if !strings.Contains(st.Detail, "avx2") {
+ t.Errorf("a CPU without AVX2 was not called out: %q", st.Detail)
+ }
+}
+
+func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) {
+ in := &Installer{Root: t.TempDir(), Host: ubuntu}
+ got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.29.0"}})
+ if got[0].State != Unsupported {
+ t.Errorf("got %+v", got[0])
+ }
+}
diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go
new file mode 100644
index 0000000..0b9910e
--- /dev/null
+++ b/internal/agent/provision/recipe.go
@@ -0,0 +1,110 @@
+// Package provision installs the engines a node is asked to have.
+//
+// The control plane says which engines at which versions; everything else is
+// worked out here, on the node, because only the node knows it: which build
+// fits its processor and driver, what its operating system is missing, and
+// whether the result actually runs. Nothing already on the machine is taken
+// on trust. An engine counts as installed when this package installed it,
+// checked every file against the checksum its publisher gave, and watched it
+// start.
+package provision
+
+import (
+ "fmt"
+ "strings"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+// Build is one published build of an engine, chosen for one machine.
+type Build struct {
+ // Variant names the build: "cpu", "cuda-12.8".
+ Variant string
+ // Assets are the release files to fetch, unpacked in order into one
+ // directory.
+ Assets []string
+ // Packages are operating system packages the build links against and the
+ // base image does not carry.
+ Packages []string
+ // Binary is the executable's file name, found anywhere in the unpacked
+ // tree.
+ Binary string
+ // CPUFlags are extensions the build is slow without. Missing ones are
+ // reported, not refused: slow is still running.
+ CPUFlags []string
+}
+
+// Recipe knows how to install one engine.
+type Recipe struct {
+ // Repo is the GitHub repository whose releases carry the builds.
+ Repo string
+ // Choose picks the build for a machine, or says why none fits.
+ Choose func(version string, h discovery.Host) (Build, error)
+}
+
+// Recipes are the engines this agent can install.
+//
+// vLLM and SGLang are not here yet: they are Python environments of several
+// gigabytes rather than an archive, and are installed by hand and named in the
+// agent's configuration until they are.
+var Recipes = map[string]Recipe{
+ "llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
+}
+
+// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
+// carries a matching cudart archive, so the node needs a driver and nothing
+// else — no toolkit.
+const cudaForLlamaCpp = "12.8"
+
+func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
+ // The published Linux builds are built on Ubuntu, and are what llama.cpp
+ // tests. Another distribution may well run them, but finding out on a
+ // production node is not the place to learn it.
+ if h.OSID != "ubuntu" {
+ return Build{}, fmt.Errorf("llama.cpp publishes builds for Ubuntu, and this node runs %s",
+ orUnknown(h.OSID))
+ }
+ var arch string
+ switch h.Arch {
+ case "amd64":
+ arch = "x64"
+ case "arm64":
+ arch = "arm64"
+ default:
+ return Build{}, fmt.Errorf("llama.cpp publishes no Ubuntu build for %s", orUnknown(h.Arch))
+ }
+
+ b := Build{
+ Variant: "cpu",
+ Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
+ // Everything else the build links is in the archive or in any Ubuntu
+ // install; OpenMP's runtime is the one a cloud image lacks, and the
+ // server does not start without it.
+ Packages: []string{"libgomp1"},
+ Binary: "llama-server",
+ }
+ if arch == "x64" {
+ // The archive carries a backend per CPU generation and loads the best
+ // one, down to SSE4.2. Below AVX2 it runs, several times slower.
+ b.CPUFlags = []string{"avx2"}
+ }
+ // The CUDA build is published for x64 at this version; on arm64 it is a
+ // newer CUDA that the driver may not support, so an arm64 GPU node runs
+ // the CPU build until someone decides otherwise.
+ if h.NVIDIADriver != "" && arch == "x64" {
+ name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
+ b.Variant = "cuda-" + cudaForLlamaCpp
+ b.Assets = []string{
+ fmt.Sprintf("llama-%s-%s", version, name),
+ fmt.Sprintf("cudart-llama-%s-%s", version, name),
+ }
+ }
+ return b, nil
+}
+
+func orUnknown(s string) string {
+ if strings.TrimSpace(s) == "" {
+ return "an unrecognised system"
+ }
+ return s
+}
diff --git a/internal/agent/selfdescribe_test.go b/internal/agent/selfdescribe_test.go
index 41afc71..5919ce4 100644
--- a/internal/agent/selfdescribe_test.go
+++ b/internal/agent/selfdescribe_test.go
@@ -1,9 +1,11 @@
package agent
import (
+ "context"
"os"
"slices"
"testing"
+ "time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
)
@@ -60,15 +62,15 @@ func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) {
engines := func(m *llmgwv1.AgentMessage) *llmgwv1.Heartbeat { return m.GetHeartbeat() }
a.mu.Lock()
- a.engineVers = map[string]string{"llamacpp": "unknown"}
- a.enginesSent = enginesFingerprint(a.engineVers)
+ a.configuredVers = map[string]string{"llamacpp": "unknown"}
+ a.enginesSent = enginesFingerprint(a.configuredVers)
a.mu.Unlock()
if hb := engines(a.heartbeat()); hb.GetEnginesChanged() {
t.Errorf("unchanged engines were resent")
}
a.mu.Lock()
- a.engineVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"}
+ a.configuredVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"}
a.mu.Unlock()
hb := engines(a.heartbeat())
if !hb.GetEnginesChanged() || len(hb.GetEngineVersions()) != 2 {
@@ -78,3 +80,40 @@ func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) {
t.Errorf("reported again on the next heartbeat")
}
}
+
+func TestWhatTheControlPlaneAsksToInstallIsReportedBack(t *testing.T) {
+ cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(),
+ EnginesDir: t.TempDir()}
+ cfg.Superv.WorkDir = t.TempDir()
+ a, err := New(cfg, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ go a.provisionLoop(ctx)
+
+ // An engine this agent cannot install: no network is needed to learn so,
+ // and the answer has to reach the control plane rather than a log.
+ a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{{Engine: "vllm", Version: "0.29.0"}}})
+
+ deadline := time.Now().Add(3 * time.Second)
+ for time.Now().Before(deadline) {
+ hb := a.heartbeat().GetHeartbeat()
+ if hb.GetProvisionChanged() {
+ p := hb.GetProvision()
+ if len(p) != 1 || p[0].GetState() != "unsupported" {
+ t.Fatalf("reported %v", p)
+ }
+ a.mu.Lock()
+ _, reported := a.engineReportLocked()["vllm"]
+ a.mu.Unlock()
+ if reported {
+ t.Error("an engine that was not installed is reported as present")
+ }
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatal("the installation's outcome never reached a heartbeat")
+}
diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go
index e2988ca..219a6d0 100644
--- a/internal/cli/ctl.go
+++ b/internal/cli/ctl.go
@@ -189,13 +189,27 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
_ = w.Flush()
// Unhealthy GPUs are usually the answer when a node looks idle but
- // nothing schedules onto it.
+ // nothing schedules onto it. An engine still installing, or one
+ // that failed to, is the other usual answer.
+ said := false
for _, n := range nodes {
for _, p := range n.Unhealthy {
fmt.Printf("\n%s: %s (kept out of placement)", n.ID, p)
+ said = true
+ }
+ for _, p := range n.Provision {
+ if p.State == "ready" {
+ continue
+ }
+ fmt.Printf("\n%s: %s %s %s: %s", n.ID, p.Engine, p.Version, p.State, p.Detail)
+ said = true
+ }
+ for _, w := range n.Warnings {
+ fmt.Printf("\n%s: %s", n.ID, w)
+ said = true
}
}
- if hasUnhealthy(nodes) {
+ if said {
fmt.Println()
}
return nil
@@ -478,12 +492,3 @@ func dash(s string) string {
}
return s
}
-
-func hasUnhealthy(nodes []adminapi.NodeView) bool {
- for _, n := range nodes {
- if len(n.Unhealthy) > 0 {
- return true
- }
- }
- return false
-}
diff --git a/internal/cli/server.go b/internal/cli/server.go
index f9ca2a4..a145d56 100644
--- a/internal/cli/server.go
+++ b/internal/cli/server.go
@@ -401,6 +401,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
}
}
svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log)
+ svc.SetEngines(cfg.WantedEngines())
+ log.Info("nodes install these engines", "engines", cfg.WantedEngines())
// A standby must refuse agents as well as refusing to place. Commands
// travel down the stream the agent opened, so a standby holding one is a
// node the leader cannot command — and the leader's placements would fail
diff --git a/internal/config/config.go b/internal/config/config.go
index c1ef662..a2e9fd6 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -72,6 +72,15 @@ type Server struct {
// Empty means there is none, and /admin/mirror stays unmounted.
Mirror Mirror `koanf:"mirror"`
+ // Engines are the engines every node installs, and at which version. The
+ // agents install them themselves — the right build for their hardware,
+ // checked against its publisher's checksum and seen to run — so a node is
+ // never trusted to already have what somebody once put there.
+ //
+ // Unset keeps DefaultEngines. A version replaces the default; an empty
+ // one leaves that engine to be installed by hand.
+ Engines map[string]string `koanf:"engines"`
+
Router Router `koanf:"router"`
Control Control `koanf:"control"`
Models []catalog.Model `koanf:"models"`
@@ -542,3 +551,25 @@ func (s Server) ControlPlaneMode() bool {
}
return false
}
+
+// DefaultEngines are the engine versions this release was checked against:
+// `llmgw ctl engine-check` accepted every flag llmgw builds for them.
+var DefaultEngines = map[string]string{
+ "llamacpp": "b10991",
+}
+
+// WantedEngines is DefaultEngines with the configuration laid over it.
+func (s Server) WantedEngines() map[string]string {
+ out := map[string]string{}
+ for k, v := range DefaultEngines {
+ out[k] = v
+ }
+ for k, v := range s.Engines {
+ if v == "" {
+ delete(out, k)
+ continue
+ }
+ out[k] = v
+ }
+ return out
+}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 0cba22e..a75eba8 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -182,3 +182,20 @@ func TestAModelWithProfilesStillMakesAControlPlane(t *testing.T) {
t.Error("a declared model with profiles is still a control plane")
}
}
+
+func TestEnginesDefaultToTheCheckedVersions(t *testing.T) {
+ var s Server
+ if got := s.WantedEngines()["llamacpp"]; got != DefaultEngines["llamacpp"] {
+ t.Errorf("unset config wants llamacpp %q", got)
+ }
+ s.Engines = map[string]string{"llamacpp": "b11000", "vllm": "0.29.0"}
+ got := s.WantedEngines()
+ if got["llamacpp"] != "b11000" || got["vllm"] != "0.29.0" {
+ t.Errorf("configured versions not taken: %v", got)
+ }
+ // Empty leaves an engine to be installed by hand.
+ s.Engines = map[string]string{"llamacpp": ""}
+ if _, ok := s.WantedEngines()["llamacpp"]; ok {
+ t.Error("an emptied engine is still wanted")
+ }
+}
diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go
index 3c5cc18..d22c645 100644
--- a/internal/control/adminapi/api.go
+++ b/internal/control/adminapi/api.go
@@ -13,6 +13,7 @@ import (
"io"
"net/http"
"slices"
+ "strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
@@ -78,6 +79,15 @@ type NodeView struct {
// which is not the same as having none — placement treats it that way too,
// or a control plane upgrade would empty a fleet.
Engines []string `json:"engines,omitempty"`
+ // Provision is where installing each wanted engine stands. An engine is in
+ // Engines only once its installation here is ready.
+ Provision []inventory.EngineInstall `json:"provision,omitempty"`
+ // OS is the operating system and architecture, as the agent read them.
+ OS string `json:"os,omitempty"`
+ // Warnings are facts about the machine that make it slow or unusable
+ // without making it unhealthy: a VM with no vector instructions runs every
+ // model, at a fifth of the speed.
+ Warnings []string `json:"warnings,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
@@ -230,6 +240,14 @@ func (a *API) Nodes() []NodeView {
v.Engines = append(v.Engines, name)
}
slices.Sort(v.Engines)
+ v.Provision = n.Provision
+ if h := n.Host; h.OSID != "" {
+ v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch)
+ if h.Arch == "amd64" && !h.HasCPUFlag("avx2") {
+ v.Warnings = append(v.Warnings, "no AVX2: inference runs several times slower "+
+ "(a VM needs CPU type \"host\")")
+ }
+ }
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go
index 6094599..0fe0e1c 100644
--- a/internal/control/agentsvc/convert.go
+++ b/internal/control/agentsvc/convert.go
@@ -23,8 +23,15 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
State: inventory.Ready,
AgentVersion: r.GetAgentVersion(),
EngineVersions: r.GetEngineVersions(),
+ EnginesProbed: r.GetEnginesProbed(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
}
+ if h := r.GetHost(); h != nil {
+ n.Host = discovery.Host{
+ OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(),
+ Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(),
+ }
+ }
for _, w := range r.GetWeights() {
n.Weights = append(n.Weights, discovery.LocalWeights{
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go
index e8678d8..db94ca0 100644
--- a/internal/control/agentsvc/service.go
+++ b/internal/control/agentsvc/service.go
@@ -49,8 +49,14 @@ type Service struct {
mu sync.RWMutex
conns map[string]*conn
+
+ // engines is what every node is told to install, set once at start.
+ engines map[string]string
}
+// SetEngines sets the engines every node installs, by name and version.
+func (s *Service) SetEngines(e map[string]string) { s.engines = maps.Clone(e) }
+
// conn is one agent's open stream.
type conn struct {
nodeID string
@@ -157,6 +163,21 @@ func (s *Service) Connect(
return err
}
+ // What this node should have installed. Sent on every registration, so a
+ // node rebuilt from nothing is told the same as one that has been here a
+ // month, and a changed version reaches every node on its next connect.
+ if s.engines != nil {
+ p := &llmgwv1.Provision{}
+ for _, e := range slices.Sorted(maps.Keys(s.engines)) {
+ p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: s.engines[e]})
+ }
+ if err := stream.Send(&llmgwv1.ControlMessage{
+ Msg: &llmgwv1.ControlMessage_Provision{Provision: p},
+ }); err != nil {
+ return err
+ }
+ }
+
// Recover replica state from what this node is already running, before any
// reconciliation pass can conclude the cluster is empty and place
// duplicates (plan §6.5).
@@ -272,7 +293,8 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
storage := hb.GetStorage()
memMB := int(hb.GetMemoryTotalMb())
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
- !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() {
+ !hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
+ !hb.GetProvisionChanged() {
return
}
@@ -339,6 +361,30 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
}
n.EngineVersions = vers
}
+ if hb.GetProvisionChanged() {
+ prev := map[string]inventory.EngineInstall{}
+ for _, p := range n.Provision {
+ prev[p.Engine] = p
+ }
+ n.Provision = n.Provision[:0]
+ for _, p := range hb.GetProvision() {
+ e := inventory.EngineInstall{
+ Engine: p.GetEngine(), Version: p.GetVersion(), Variant: p.GetVariant(),
+ State: p.GetState(), Detail: p.GetDetail(),
+ }
+ // A state change is worth a line; a download's progress is not.
+ if old, ok := prev[e.Engine]; !ok || old.State != e.State {
+ lvl := slog.LevelInfo
+ if e.State == "failed" || e.State == "unsupported" {
+ lvl = slog.LevelWarn
+ }
+ s.log.Log(context.Background(), lvl, "engine installation on a node",
+ "node", nodeID, "engine", e.Engine, "version", e.Version,
+ "variant", e.Variant, "state", e.State, "detail", e.Detail)
+ }
+ n.Provision = append(n.Provision, e)
+ }
+ }
s.inv.Upsert(n)
}
diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go
index ece7595..4617286 100644
--- a/internal/control/inventory/inventory.go
+++ b/internal/control/inventory/inventory.go
@@ -102,6 +102,24 @@ type Node struct {
// version decides which metric names and CLI flags apply (plan §14).
AgentVersion string `json:"agent_version,omitempty"`
EngineVersions map[string]string `json:"engine_versions,omitempty"`
+ // EnginesProbed says EngineVersions is an answer, so an empty map means
+ // "has none" rather than "an agent too old to say".
+ EnginesProbed bool `json:"engines_probed,omitempty"`
+
+ // Host is the operating system and processor, as the agent reads them.
+ Host discovery.Host `json:"host"`
+ // Provision is where installing each wanted engine stands on this node.
+ Provision []EngineInstall `json:"provision,omitempty"`
+}
+
+// EngineInstall is where one engine's installation stands on a node.
+type EngineInstall struct {
+ Engine string `json:"engine"`
+ Version string `json:"version"`
+ Variant string `json:"variant,omitempty"`
+ // State is "installing", "ready", "failed" or "unsupported".
+ State string `json:"state"`
+ Detail string `json:"detail,omitempty"`
}
// HasFabric reports whether the node is attached to a fabric.
diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go
index 60af93b..b10d776 100644
--- a/internal/control/placement/placement.go
+++ b/internal/control/placement/placement.go
@@ -403,9 +403,14 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
// A node that reports no engines at all is not excluded. That is what
// every agent older than this reports, and emptying a fleet on a
// control plane upgrade is a worse failure than the one being fixed.
- if p.Engine != "" && len(n.EngineVersions) > 0 {
+ //
+ // An agent that says it probed is believed when it reports nothing:
+ // that is a node still installing, or one where installing failed,
+ // and either way not a place to start an engine. sim is built into
+ // the agent and never installed.
+ if p.Engine != "" && p.Engine != "sim" && (n.EnginesProbed || len(n.EngineVersions) > 0) {
if _, ok := n.EngineVersions[p.Engine]; !ok {
- reject("does not have " + p.Engine + " installed")
+ reject("does not have " + p.Engine + " installed" + installing(n, p.Engine))
continue
}
}
@@ -526,3 +531,13 @@ func (s *Scheduler) claimedMemory(nodeID string) int {
}
return total
}
+
+// installing says why an engine is missing, when the node has said.
+func installing(n inventory.Node, engine string) string {
+ for _, p := range n.Provision {
+ if p.Engine == engine && p.Detail != "" {
+ return " (" + p.State + ": " + p.Detail + ")"
+ }
+ }
+ return ""
+}
diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go
index 13f7f0e..9a241d6 100644
--- a/internal/control/placement/placement_test.go
+++ b/internal/control/placement/placement_test.go
@@ -797,3 +797,33 @@ func TestAProfileThatDeclaresNothingClaimsNothing(t *testing.T) {
t.Errorf("an undeclared neighbour should not fill the node:\n%s", res.Explain())
}
}
+
+func TestANodeStillInstallingItsEngineIsNotACandidate(t *testing.T) {
+ // A node rebuilt from nothing registers before its agent has installed
+ // anything, and reports no engines. Treated as "too old to say", it was a
+ // candidate for every model at the moment it could run none of them.
+ fresh := node("fresh", "h100-80g", 8)
+ fresh.EnginesProbed = true
+ fresh.Provision = []inventory.EngineInstall{{
+ Engine: "vllm", Version: "0.29.0", State: "installing", Detail: "downloading",
+ }}
+
+ res := New(store(fresh)).Place([]Profile{{
+ Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1,
+ }}, nil)
+ if res.Placement != nil {
+ t.Fatal("placed on a node whose engine is not installed yet")
+ }
+ if !strings.Contains(res.Explain(), "installing: downloading") {
+ t.Errorf("the rejection should say the engine is on its way:\n%s", res.Explain())
+ }
+
+ // The simulated engine is part of the agent, so it is never installed and
+ // never missing.
+ res = New(store(fresh)).Place([]Profile{{
+ Name: "sim", Engine: "sim", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1,
+ }}, nil)
+ if res.Placement == nil {
+ t.Errorf("sim was refused:\n%s", res.Explain())
+ }
+}
diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html
index ef796c8..045dce3 100644
--- a/internal/web/templates/dashboard.html
+++ b/internal/web/templates/dashboard.html
@@ -101,7 +101,7 @@ requested whenever a fallback served it.
Nodes
{{if .Nodes}}
-| Node | State | GPUs | RAM | Disk | Classes | Fabrics | Models | Replicas | Last seen |
+| Node | State | GPUs | RAM | Disk | Engines | Classes | Fabrics | Models | Replicas | Last seen |
{{range .Nodes}}
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} |
{{.State}} |
@@ -111,12 +111,15 @@ requested whenever a fallback served it.
second, and an em dash where nothing was reported: an agent that
was never asked about its disk must not read as a full one. */}}
{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}} |
+ {{if .Engines}}{{join .Engines}}{{else}}none yet{{end}} |
{{join .Classes}} |
{{join .Fabrics}} |
{{if .Models}}{{len .Models}}{{else}}—{{end}} |
{{.Replicas}} |
{{or .LastSeenAgo "—"}} |
-
{{if .Unhealthy}}| {{join .Unhealthy}} |
{{end}}
+{{if .Unhealthy}}| {{join .Unhealthy}} |
{{end}}
+{{range .Provision}}{{if ne .State "ready"}}| {{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}} |
{{end}}{{end}}
+{{range .Warnings}}| {{.}} |
{{end}}
{{end}}
{{else}}No agents have registered.
{{end}}