From 774e134ed159e8c1a5e85f27c733ac5d24ee6ce8 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 18 Sep 2026 09:51:05 +0200 Subject: [PATCH] Refuse a node with no room, and take back only what llmgw put there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the same problem: nothing stopped a model being placed on a node that could not hold it, and nothing ever removed the weights of a model that had been deleted from the catalog. Placement now has a disk check, shaped exactly like the memory one it sits beside, with one addition: it is skipped on a node that already holds the weights. A node holding the file needs no room for it, and without that the node best placed to run a model — because it already has it — is the one most likely to be rejected, since holding the file is what used the space. Reclaiming is the dangerous half, so the whole design is about what it will not do. llmgw deletes only files it fetched itself, and that is a calculation rather than a heuristic: the cache name is a hash of the source, so the set of wanted files can be computed from the catalog exactly. Anything without that name was put there by an operator — scp, a shared mount, a copy by hand — and is none of this system's business however unreferenced it looks. A model that took a week to copy must not be removable by a command someone ran to tidy up. The node checks every path again before unlinking it, and that duplication is the point. The control plane knows the catalog; the node knows its disk. A control plane that is wrong, newer than the agent, or tampered with must not be able to talk a node into deleting something, so the agent refuses anything outside its own cache directory or without the cache name — including a path that tries to traverse out of it. Listing is the default and deleting takes --collect. The summary says "asked to remove", because each node decides, and one that refuses a path it cannot prove it fetched is this working rather than failing. Co-Authored-By: Claude Opus 5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 155 +++++++++++++++++------- api/proto/llmgw/v1/agent.proto | 12 ++ internal/agent/agent.go | 58 +++++++++ internal/agent/removeweights_test.go | 96 +++++++++++++++ internal/agent/weights/fetch.go | 9 +- internal/agent/weights/gc.go | 99 +++++++++++++++ internal/agent/weights/gc_test.go | 116 ++++++++++++++++++ internal/cli/ctl.go | 1 + internal/cli/ctlweights.go | 153 +++++++++++++++++++++++ internal/control/adminapi/api.go | 4 + internal/control/adminapi/weights.go | 107 ++++++++++++++++ internal/control/agentsvc/service.go | 16 +++ internal/control/catalog/catalog.go | 15 +++ internal/control/control.go | 17 +++ internal/control/placement/disk_test.go | 93 ++++++++++++++ internal/control/placement/placement.go | 19 +++ 16 files changed, 921 insertions(+), 49 deletions(-) create mode 100644 internal/agent/removeweights_test.go create mode 100644 internal/agent/weights/gc.go create mode 100644 internal/agent/weights/gc_test.go create mode 100644 internal/cli/ctlweights.go create mode 100644 internal/control/adminapi/weights.go create mode 100644 internal/control/placement/disk_test.go diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d6d8ecf..3ff8981 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -201,6 +201,7 @@ type ControlMessage struct { // *ControlMessage_Stop // *ControlMessage_Drain // *ControlMessage_Ack + // *ControlMessage_RemoveWeights Msg isControlMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -279,6 +280,15 @@ func (x *ControlMessage) GetAck() *Ack { return nil } +func (x *ControlMessage) GetRemoveWeights() *RemoveWeights { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_RemoveWeights); ok { + return x.RemoveWeights + } + } + return nil +} + type isControlMessage_Msg interface { isControlMessage_Msg() } @@ -299,6 +309,10 @@ type ControlMessage_Ack struct { Ack *Ack `protobuf:"bytes,4,opt,name=ack,proto3,oneof"` } +type ControlMessage_RemoveWeights struct { + RemoveWeights *RemoveWeights `protobuf:"bytes,5,opt,name=remove_weights,json=removeWeights,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -307,6 +321,8 @@ func (*ControlMessage_Drain) isControlMessage_Msg() {} func (*ControlMessage_Ack) isControlMessage_Msg() {} +func (*ControlMessage_RemoveWeights) isControlMessage_Msg() {} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. @@ -1446,6 +1462,57 @@ func (x *EngineProfile) GetServedModelName() string { return "" } +// RemoveWeights deletes cached model files a node no longer needs. +// +// The paths are computed by the control plane from the catalog, but the agent +// checks them again before unlinking anything: it deletes only files inside its +// own cache directory whose names this system generated. A control plane that +// is wrong, compromised or simply newer than the agent must not be able to +// talk it into deleting a model an operator spent a week copying there. +type RemoveWeights struct { + state protoimpl.MessageState `protogen:"open.v1"` + Paths []string `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RemoveWeights) Reset() { + *x = RemoveWeights{} + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RemoveWeights) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveWeights) ProtoMessage() {} + +func (x *RemoveWeights) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. +func (*RemoveWeights) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} +} + +func (x *RemoveWeights) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + // StopShard terminates a shard and frees its GPUs. type StopShard struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1456,7 +1523,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1468,7 +1535,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1481,7 +1548,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *StopShard) GetShardId() string { @@ -1501,7 +1568,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1513,7 +1580,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1526,7 +1593,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *DrainShard) GetShardId() string { @@ -1548,7 +1615,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1560,7 +1627,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1573,7 +1640,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *Ack) GetMessageId() string { @@ -1606,12 +1673,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" + "\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" + "\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shardB\x05\n" + - "\x03msg\"\xc1\x01\n" + + "\x03msg\"\x83\x02\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + - "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" + + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ack\x12@\n" + + "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeightsB\x05\n" + "\x03msg\"\xa7\x05\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + @@ -1748,7 +1816,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"&\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" + + "\rRemoveWeights\x12\x14\n" + + "\x05paths\x18\x01 \x03(\tR\x05paths\"&\n" + "\tStopShard\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\"'\n" + "\n" + @@ -1777,7 +1847,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -1792,39 +1862,41 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus (*StartShard)(nil), // 11: llmgw.v1.StartShard (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile - (*StopShard)(nil), // 13: llmgw.v1.StopShard - (*DrainShard)(nil), // 14: llmgw.v1.DrainShard - (*Ack)(nil), // 15: llmgw.v1.Ack - nil, // 16: llmgw.v1.Register.LabelsEntry - nil, // 17: llmgw.v1.Register.EngineVersionsEntry - nil, // 18: llmgw.v1.EngineProfile.EnvEntry + (*RemoveWeights)(nil), // 13: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 14: llmgw.v1.StopShard + (*DrainShard)(nil), // 15: llmgw.v1.DrainShard + (*Ack)(nil), // 16: llmgw.v1.Ack + nil, // 17: llmgw.v1.Register.LabelsEntry + nil, // 18: llmgw.v1.Register.EngineVersionsEntry + nil, // 19: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 13, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 14, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 15, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 16, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 6, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 7, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 17, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 5, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 8, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 4, // 13: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 0, // 14: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 0, // 15: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 12, // 16: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 18, // 17: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 18: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 19: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 19, // [19:20] is the sub-list for method output_type - 18, // [18:19] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 18, // [18:18] is the sub-list for extension extendee - 0, // [0:18] is the sub-list for field type_name + 14, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 15, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 16, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 13, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 17, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 6, // 9: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 7, // 10: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 18, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 5, // 12: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 0, // 16: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 12, // 17: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 19, // 18: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 19: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 20: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 20, // [20:21] is the sub-list for method output_type + 19, // [19:20] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1842,6 +1914,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_Stop)(nil), (*ControlMessage_Drain)(nil), (*ControlMessage_Ack)(nil), + (*ControlMessage_RemoveWeights)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -1849,7 +1922,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 18, + NumMessages: 19, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 8fc6c2f..f4792bf 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -33,6 +33,7 @@ message ControlMessage { StopShard stop = 2; DrainShard drain = 3; Ack ack = 4; + RemoveWeights remove_weights = 5; } } @@ -266,6 +267,17 @@ message EngineProfile { string served_model_name = 11; } +// RemoveWeights deletes cached model files a node no longer needs. +// +// The paths are computed by the control plane from the catalog, but the agent +// checks them again before unlinking anything: it deletes only files inside its +// own cache directory whose names this system generated. A control plane that +// is wrong, compromised or simply newer than the agent must not be able to +// talk it into deleting a model an operator spent a week copying there. +message RemoveWeights { + repeated string paths = 1; +} + // StopShard terminates a shard and frees its GPUs. message StopShard { string shard_id = 1; diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 458139d..c5f36d2 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -11,6 +11,8 @@ import ( "math/rand/v2" "net" "net/http" + "os" + "path/filepath" "strings" "sync" "time" @@ -581,6 +583,60 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}} } +// removeWeights deletes cached model files the control plane says are no +// longer needed. +// +// Every path is checked here as well as there, and that duplication is the +// point. The control plane computes these from the catalog, but it is the node +// that holds the files, and a node must not be talkable into deleting a model +// an operator spent a week copying onto it — not by a control plane that is +// wrong, not by one that is newer than this agent, not by one that has been +// tampered with. So this deletes only what it can prove llmgw itself wrote: +// inside its own cache directory, under a name this system generates. +// +// Failures are logged and skipped rather than reported upward. A file that is +// already gone is the expected outcome of a retry, and one that will not +// delete is a disk to look at, not a reason to fail a command whose purpose +// was to free space. +func (a *Agent) removeWeights(paths []string) { + cache := a.cfg.WeightsCache + if cache == "" && len(a.cfg.ModelDirs) > 0 { + cache = a.cfg.ModelDirs[0] + } + if cache == "" { + a.log.Warn("asked to remove weights with no cache directory configured") + return + } + cacheAbs, err := filepath.Abs(cache) + if err != nil { + a.log.Warn("cannot resolve the cache directory", "dir", cache, "err", err) + return + } + + for _, p := range paths { + abs, err := filepath.Abs(p) + if err != nil { + continue + } + // Directly inside the cache directory, not merely under it: a nested + // path would let ".." or a symlinked subdirectory reach elsewhere, and + // nothing this package writes is nested anyway. + if filepath.Dir(abs) != cacheAbs { + a.log.Warn("refused to remove a file outside the weights cache", "path", p) + continue + } + if !weights.Owned(abs) { + a.log.Warn("refused to remove a file llmgw did not fetch", "path", p) + continue + } + if err := os.Remove(abs); err != nil && !os.IsNotExist(err) { + a.log.Warn("could not remove cached weights", "path", p, "err", err) + continue + } + a.log.Info("removed cached weights no model asks for", "path", p) + } +} + func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) { switch m := msg.Msg.(type) { case *llmgwv1.ControlMessage_Start: @@ -591,6 +647,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha if err := a.sup.Drain(m.Drain.GetShardId()); err != nil { a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err) } + case *llmgwv1.ControlMessage_RemoveWeights: + a.removeWeights(m.RemoveWeights.GetPaths()) case *llmgwv1.ControlMessage_Ack: if !m.Ack.GetOk() { a.log.Warn("control plane rejected a message", diff --git a/internal/agent/removeweights_test.go b/internal/agent/removeweights_test.go new file mode 100644 index 0000000..107a3de --- /dev/null +++ b/internal/agent/removeweights_test.go @@ -0,0 +1,96 @@ +package agent + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" +) + +// agentWithCache is an agent that only knows where its weights live, which is +// all removeWeights needs. +func agentWithCache(t *testing.T, dir string) *Agent { + t.Helper() + return &Agent{ + cfg: Config{ModelDirs: []string{dir}}, + log: slog.New(slog.DiscardHandler), + } +} + +func write(t *testing.T, path string) string { + t.Helper() + if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestTheAgentRefusesToDeleteWhatItDidNotFetch(t *testing.T) { + // The control plane computes these paths, but the node holds the files. A + // control plane that is wrong, newer than this agent, or tampered with must + // not be able to talk it into deleting a model an operator spent a week + // copying here. So the agent checks again, on its own. + dir := t.TempDir() + a := agentWithCache(t, dir) + + operators := write(t, filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0.gguf")) + fetched := write(t, weights.CachePathFor(dir, "https://models.internal/gone.gguf")) + + a.removeWeights([]string{operators, fetched}) + + if _, err := os.Stat(operators); err != nil { + t.Errorf("a file llmgw never fetched was deleted: %v", err) + } + if _, err := os.Stat(fetched); !os.IsNotExist(err) { + t.Errorf("the fetched file was not removed: %v", err) + } +} + +func TestTheAgentRefusesPathsOutsideItsCache(t *testing.T) { + // Even with a name that looks like one of ours. The cache directory is the + // boundary, and traversal out of it is the attack this closes. + dir := t.TempDir() + elsewhere := t.TempDir() + a := agentWithCache(t, dir) + + // A correctly named cache file, in the wrong directory. + outside := write(t, weights.CachePathFor(elsewhere, "https://models.internal/x.gguf")) + // And the same thing reached by traversal from inside the cache. + traversal := filepath.Join(dir, "..", filepath.Base(elsewhere), + filepath.Base(outside)) + + a.removeWeights([]string{outside, traversal}) + + if _, err := os.Stat(outside); err != nil { + t.Errorf("a file outside the cache was deleted: %v", err) + } +} + +func TestRemovingAFileThatIsAlreadyGoneIsNotAFailure(t *testing.T) { + // The expected outcome of a retry. It must not stop the rest of the batch. + dir := t.TempDir() + a := agentWithCache(t, dir) + + missing := weights.CachePathFor(dir, "https://models.internal/never.gguf") + present := write(t, weights.CachePathFor(dir, "https://models.internal/here.gguf")) + + a.removeWeights([]string{missing, present}) + + if _, err := os.Stat(present); !os.IsNotExist(err) { + t.Errorf("a missing file stopped the batch: %v", err) + } +} + +func TestWithNoCacheConfiguredNothingIsDeleted(t *testing.T) { + dir := t.TempDir() + a := &Agent{cfg: Config{}, log: slog.New(slog.DiscardHandler)} + kept := write(t, weights.CachePathFor(dir, "https://models.internal/x.gguf")) + + a.removeWeights([]string{kept}) + + if _, err := os.Stat(kept); err != nil { + t.Errorf("an agent with no cache directory deleted something: %v", err) + } +} diff --git a/internal/agent/weights/fetch.go b/internal/agent/weights/fetch.go index 3d07017..09ea796 100644 --- a/internal/agent/weights/fetch.go +++ b/internal/agent/weights/fetch.go @@ -15,8 +15,6 @@ package weights import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" "io" @@ -262,12 +260,7 @@ func (f *Fetcher) cachePath(source string) (string, error) { if f.CacheDir == "" { return "", fmt.Errorf("no cache directory configured, so %s cannot be fetched", source) } - sum := sha256.Sum256([]byte(source)) - name := filepath.Base(source) - if name == "" || name == "." || name == "/" { - name = "weights" - } - return filepath.Join(f.CacheDir, hex.EncodeToString(sum[:8])+"-"+name), nil + return CachePathFor(f.CacheDir, source), nil } // isPath reports whether a source names a file on this machine. diff --git a/internal/agent/weights/gc.go b/internal/agent/weights/gc.go new file mode 100644 index 0000000..9c64b5e --- /dev/null +++ b/internal/agent/weights/gc.go @@ -0,0 +1,99 @@ +package weights + +import ( + "crypto/sha256" + "encoding/hex" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// cacheNamePattern matches a filename this package created. +// +// The eight-byte hash of the source, hex encoded, then a dash, then the +// original filename — exactly what cachePath writes. It is the only mark of +// ownership there is, and it has to be exact: everything else in a model +// directory was put there by the operator, and deleting one of those files +// could cost a week of someone's bandwidth. +var cacheNamePattern = regexp.MustCompile(`^[0-9a-f]{16}-`) + +// CachePathFor is where a source would be cached in the given directory. +// +// Exported so the control plane can work out which files in a node's reported +// list are ones llmgw fetched for a model it still wants, without asking the +// node anything. The agent and the control plane must agree on this name +// exactly, which is why there is one function and not two. +func CachePathFor(dir, source string) string { + sum := sha256.Sum256([]byte(source)) + name := filepath.Base(source) + if name == "" || name == "." || name == "/" { + name = "weights" + } + return filepath.Join(dir, hex.EncodeToString(sum[:8])+"-"+name) +} + +// Owned reports whether a path is one this package created. +// +// The dividing line of the whole feature. A file without the cache name was +// put there by the operator — by scp, by a shared mount, by hand — and llmgw +// has no business deleting it however unreferenced it looks. A file with the +// cache name is one llmgw downloaded, and llmgw can take it back. +func Owned(path string) bool { + return cacheNamePattern.MatchString(filepath.Base(path)) +} + +// Orphans lists cached files that no wanted source accounts for. +// +// `have` is what a node reports on disk and `wanted` is every source in the +// catalog; both are passed in so this is a calculation rather than a scan, and +// so it can be run on the control plane against a node's reported inventory. +// +// Only files this package created are ever returned. A source that is not +// fetched — a plain path, or a model id an engine resolves itself — produces +// no cache name and so protects nothing and deletes nothing, which is correct: +// those files are not in the cache to begin with. +func Orphans(have []string, wanted []string) []string { + keep := map[string]struct{}{} + for _, src := range wanted { + if !isRemote(src) { + // Nothing was fetched for it, so nothing in the cache belongs to + // it. Computing a name here would reserve a file that does not + // exist and could never be an orphan anyway. + continue + } + // By base name rather than full path: the control plane does not know + // each node's cache directory, and the name is what identifies the + // file wherever the node keeps it. + keep[filepath.Base(CachePathFor("", src))] = struct{}{} + } + + var out []string + seen := map[string]struct{}{} + for _, path := range have { + base := filepath.Base(path) + if !Owned(base) { + continue + } + if _, wanted := keep[base]; wanted { + continue + } + if _, dup := seen[path]; dup { + continue + } + seen[path] = struct{}{} + out = append(out, path) + } + sort.Strings(out) + return out +} + +// PartialName reports whether a path is the temporary file a fetch writes to. +// +// Those are not orphans in the same sense — a fetch may be running right now — +// so they are named separately rather than swept up with the rest. A `.part` +// left behind by a killed agent is real garbage, but only its own node can +// tell whether it is stale, and nothing here can. +func PartialName(path string) bool { + return strings.HasSuffix(path, ".part") +} diff --git a/internal/agent/weights/gc_test.go b/internal/agent/weights/gc_test.go new file mode 100644 index 0000000..aaef840 --- /dev/null +++ b/internal/agent/weights/gc_test.go @@ -0,0 +1,116 @@ +package weights + +import ( + "path/filepath" + "slices" + "testing" +) + +func TestOnlyFilesLLMGWFetchedAreEverOrphans(t *testing.T) { + // The invariant the whole feature rests on. Everything without the cache + // name was put there by the operator — scp, a shared mount, a copy by hand + // — and deleting one could cost a week of someone's bandwidth. It does not + // matter how unreferenced it looks. + const dir = "/var/lib/llmgw-models" + have := []string{ + filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0-00001-of-00002.gguf"), + filepath.Join(dir, "some-model.safetensors"), + filepath.Join(dir, "notes.txt"), + CachePathFor(dir, "https://models.internal/gone.gguf"), + } + + got := Orphans(have, nil) + if len(got) != 1 { + t.Fatalf("expected only the fetched file, got %v", got) + } + if !Owned(got[0]) { + t.Errorf("returned a file llmgw did not fetch: %s", got[0]) + } +} + +func TestAFetchedFileStillInTheCatalogIsKept(t *testing.T) { + const dir = "/var/lib/llmgw-models" + const wanted = "hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf" + const gone = "https://models.internal/old.gguf" + + have := []string{CachePathFor(dir, wanted), CachePathFor(dir, gone)} + got := Orphans(have, []string{wanted}) + + if slices.Contains(got, CachePathFor(dir, wanted)) { + t.Error("a file the catalog still points at was called an orphan") + } + if !slices.Contains(got, CachePathFor(dir, gone)) { + t.Errorf("a file nothing points at was not found: %v", got) + } +} + +func TestTheCacheNameMatchesWhatAFetchActuallyWrites(t *testing.T) { + // The control plane proposes deletions by recomputing this name from the + // catalog. If it ever drifted from what a fetch writes, the proposal would + // name a file that is in use — so the two must be one function, and this + // is what says so. + f := &Fetcher{CacheDir: "/cache"} + const source = "https://models.internal/x.gguf" + + got, err := f.cachePath(source) + if err != nil { + t.Fatal(err) + } + if want := CachePathFor("/cache", source); got != want { + t.Errorf("a fetch writes %q but the catalog would look for %q", got, want) + } +} + +func TestOneOfTwoLookalikeFilesIsNotMistakenForTheOther(t *testing.T) { + // That the two cache to different names is covered where the naming lives. + // What matters here is the consequence: deleting the unreferenced one must + // not take the referenced one with it, and both end in "model.gguf". + a := CachePathFor("/cache", "https://one.example/model.gguf") + b := CachePathFor("/cache", "https://two.example/model.gguf") + + got := Orphans([]string{a, b}, []string{"https://one.example/model.gguf"}) + if !slices.Equal(got, []string{b}) { + t.Errorf("orphans = %v, want only %s", got, b) + } +} + +func TestASourceThatIsNeverFetchedProtectsNothingAndDeletesNothing(t *testing.T) { + // A plain path, or a model id an engine resolves itself, puts nothing in + // the cache. Computing a cache name for it would reserve a file that does + // not exist, and it must not stop a real orphan being found either. + const dir = "/models" + orphan := CachePathFor(dir, "https://models.internal/gone.gguf") + + got := Orphans([]string{orphan, filepath.Join(dir, "local.gguf")}, + []string{"/models/local.gguf", "Qwen/Qwen2.5-7B-Instruct"}) + + if !slices.Equal(got, []string{orphan}) { + t.Errorf("orphans = %v, want only %s", got, orphan) + } +} + +func TestAnEmptyCatalogDoesNotMakeOperatorFilesDisposable(t *testing.T) { + // The dangerous case: every model removed from the catalog at once. Files + // llmgw fetched become orphans, which is right. Files it did not must stay + // untouched, which is the whole point. + const dir = "/models" + have := []string{ + filepath.Join(dir, "hand-placed.gguf"), + CachePathFor(dir, "https://models.internal/a.gguf"), + } + got := Orphans(have, nil) + if len(got) != 1 || got[0] != CachePathFor(dir, "https://models.internal/a.gguf") { + t.Errorf("orphans = %v", got) + } +} + +func TestPartialsAreNotSweptUpWithTheRest(t *testing.T) { + // A fetch may be running right now, and only the node itself can tell a + // live download from one a killed agent left behind. + if !PartialName("/models/abc-model.gguf.part") { + t.Error("a .part file was not recognised") + } + if PartialName("/models/abc-model.gguf") { + t.Error("a finished file was called partial") + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index b98c888..1a9e637 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -126,6 +126,7 @@ func newCtlCmd() *cobra.Command { newModelCmd(client), newEngineCheckCmd(), newToolCheckCmd(), + newWeightsCmd(client, &asJSON), newCertCmd(), ) return cmd diff --git a/internal/cli/ctlweights.go b/internal/cli/ctlweights.go new file mode 100644 index 0000000..786e124 --- /dev/null +++ b/internal/cli/ctlweights.go @@ -0,0 +1,153 @@ +package cli + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// newWeightsCmd reports and reclaims model files nothing needs. +// +// The dangerous half of storage management, so its shape is defensive: listing +// is the default and deleting takes a flag, only files llmgw fetched are ever +// candidates, and the node checks every path again before unlinking it. +// +// What it will never touch is the more important half of the contract. Weights +// an operator put on a node — by scp, by a shared mount, by hand — are none of +// this system's business however unreferenced they look, and a model that took +// a week to copy must not be removable by a command someone ran to tidy up. +func newWeightsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + cmd := &cobra.Command{ + Use: "weights", + Short: "Show cached model files no model in the catalog needs", + Long: "Lists weights llmgw downloaded for models that are no longer in the\n" + + "catalog, per node, with what deleting them would free.\n\n" + + "Only files llmgw fetched are considered. Anything an operator put on a\n" + + "node is left alone, whatever the catalog says — llmgw observes storage\n" + + "rather than owning it, and only takes back what it put there itself.\n\n" + + "Nothing is deleted without --collect.", + Args: cobra.NoArgs, + } + var collect bool + var node string + + cmd.Flags().BoolVar(&collect, "collect", false, "delete them, rather than only listing") + cmd.Flags().StringVar(&node, "node", "", "limit to one node") + + cmd.RunE = func(cmd *cobra.Command, _ []string) error { + if collect { + return collectOrphans(cmd, client(), node, *asJSON) + } + return listOrphanWeights(cmd, client(), node, *asJSON) + } + return cmd +} + +func listOrphanWeights(cmd *cobra.Command, c *ctlClient, node string, asJSON bool) error { + var found []adminapi.OrphanView + if err := c.get(cmd.Context(), "/api/v1/weights/orphans", &found); err != nil { + return err + } + if node != "" { + found = onlyNode(found, node) + } + if asJSON { + return emitJSON(found) + } + if len(found) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), + "Nothing to reclaim: every file llmgw fetched is still wanted by a model.") + return nil + } + + w := tab() + fmt.Fprintln(w, "NODE\tFILES\tWOULD FREE") + var total int64 + for _, o := range found { + fmt.Fprintf(w, "%s\t%d\t%s\n", o.Node, len(o.Paths), fileSize(o.Bytes)) + total += o.Bytes + } + _ = w.Flush() + + for _, o := range found { + for _, p := range o.Paths { + fmt.Fprintf(cmd.OutOrStdout(), "\n %s: %s", o.Node, p) + } + } + fmt.Fprintf(cmd.OutOrStdout(), "\n\n%s in total. Delete with --collect.\n", fileSize(total)) + return nil +} + +func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool) error { + body, err := json.Marshal(map[string]string{"node": node}) + if err != nil { + return err + } + var results []struct { + Node string `json:"node"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` + } + if err := c.post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil { + return err + } + if asJSON { + return emitJSON(results) + } + if len(results) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "Nothing to reclaim.") + return nil + } + var freed int64 + for _, r := range results { + if r.Error != "" { + // Reported rather than fatal: the other nodes' space is still worth + // freeing, and this one will be offered again next time. + fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", r.Node, r.Error) + continue + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: asked to remove %d files, %s\n", + r.Node, r.Files, fileSize(r.Bytes)) + freed += r.Bytes + } + // "asked", not "removed": each node decides, and one that refuses a path it + // cannot prove it fetched is the feature working rather than failing. + fmt.Fprintf(cmd.OutOrStdout(), "\n%s requested. The nodes verify each path before deleting it.\n", + fileSize(freed)) + return nil +} + +func onlyNode(in []adminapi.OrphanView, node string) []adminapi.OrphanView { + out := make([]adminapi.OrphanView, 0, len(in)) + for _, o := range in { + if o.Node == node { + out = append(out, o) + } + } + return out +} + +// fileSize renders a size the way someone reads one when deciding whether a +// file is worth deleting. +// +// Separate from the per-token cache size renderer in modelplan, which never +// leaves KiB and MiB. Model files are tens of gigabytes, and "8601.6 MiB" is +// not a number anyone weighs a decision against. +func fileSize(b int64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.1f GB", float64(b)/(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.0f MB", float64(b)/(1<<20)) + case b > 0: + return fmt.Sprintf("%d B", b) + default: + // The node reported no size. Saying "0 B" would suggest deleting it + // gains nothing, which is a different claim from not knowing. + return "size unknown" + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 2b9792a..6024ad5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -40,6 +40,10 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/nodes", a.listNodes) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) + // Listing is a read and deleting is a separate verb, so a mistyped + // command cannot remove anything. + mux.HandleFunc("GET /api/v1/weights/orphans", a.listOrphans) + mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights) mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement) mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState) } diff --git a/internal/control/adminapi/weights.go b/internal/control/adminapi/weights.go new file mode 100644 index 0000000..2f7927b --- /dev/null +++ b/internal/control/adminapi/weights.go @@ -0,0 +1,107 @@ +package adminapi + +import ( + "encoding/json" + "net/http" + "sort" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" +) + +// OrphanView is cached weights on one node that no catalog model accounts for. +type OrphanView struct { + Node string `json:"node"` + Paths []string `json:"paths"` + // Bytes is what deleting them would free, as far as the node reported. A + // file whose size was not reported counts as zero rather than being left + // out, so the list and the number always describe the same set. + Bytes int64 `json:"bytes"` +} + +// Orphans finds cached weights nothing in the catalog points at. +// +// Computed here from what nodes already report, so listing costs no round trip +// and a node that is offline simply contributes nothing — its files are not +// deletable while it is gone anyway. +// +// Only files llmgw fetched are ever considered. Everything else in a model +// directory was put there by the operator and is none of this system's +// business, however unreferenced it looks. +func (a *API) Orphans() []OrphanView { + var wanted []string + for _, m := range a.ctl.Catalog().Models() { + wanted = append(wanted, m.Source) + } + + out := make([]OrphanView, 0) + for _, n := range a.ctl.Inventory().Nodes() { + size := map[string]int64{} + have := make([]string, 0, len(n.Weights)) + for _, w := range n.Weights { + have = append(have, w.Path) + size[w.Path] = w.SizeBytes + } + + paths := weights.Orphans(have, wanted) + if len(paths) == 0 { + continue + } + v := OrphanView{Node: n.ID, Paths: paths} + for _, p := range paths { + v.Bytes += size[p] + } + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) + return out +} + +func (a *API) listOrphans(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.Orphans()) +} + +// collectWeights deletes them. +// +// A proposal, not an instruction: each node checks every path again before +// unlinking it, and refuses anything it cannot prove it fetched. The two +// checks are deliberate duplication — the control plane knows the catalog and +// the node knows its disk, and only the node can be sure. +func (a *API) collectWeights(w http.ResponseWriter, r *http.Request) { + rm := a.ctl.WeightRemover() + if rm == nil { + writeError(w, http.StatusNotImplemented, + "this control plane cannot ask nodes to delete files") + return + } + + var body struct { + // Node limits it to one node. Empty means every node, which is the + // common case after removing a model from the catalog. + Node string `json:"node,omitempty"` + } + // An empty body is the "all nodes" case and must not be an error. + _ = json.NewDecoder(r.Body).Decode(&body) + + type result struct { + Node string `json:"node"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` + } + out := make([]result, 0) + for _, o := range a.Orphans() { + if body.Node != "" && o.Node != body.Node { + continue + } + res := result{Node: o.Node, Files: len(o.Paths), Bytes: o.Bytes} + // A node that has gone since the list was built is reported rather than + // failing the whole sweep: the other nodes' space is still worth + // freeing, and this one will be offered again next time. + if err := rm.RemoveWeights(r.Context(), o.Node, o.Paths); err != nil { + res.Error = err.Error() + res.Files, res.Bytes = 0, 0 + } + out = append(out, res) + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index de2b7c5..3309028 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -340,6 +340,22 @@ func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error { }) } +// RemoveWeights asks a node to delete cached files no model needs. +// +// The node checks every path again before unlinking it, so this is a proposal +// rather than an instruction — which is the right shape for a message that +// deletes data the control plane cannot see. +func (s *Service) RemoveWeights(_ context.Context, nodeID string, paths []string) error { + if len(paths) == 0 { + return nil + } + return s.send(nodeID, &llmgwv1.ControlMessage{ + Msg: &llmgwv1.ControlMessage_RemoveWeights{ + RemoveWeights: &llmgwv1.RemoveWeights{Paths: paths}, + }, + }) +} + func (s *Service) send(nodeID string, msg *llmgwv1.ControlMessage) error { s.mu.RLock() c, ok := s.conns[nodeID] diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 7a17ce6..644fbff 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -71,6 +71,20 @@ type Profile struct { // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` + // MinDiskGB is the free space a node needs before it can hold these + // weights. + // + // Only checked when the node does not already have them: a node holding + // the file needs no room for it, and requiring the space twice would keep + // a model off the one node best placed to run it. + // + // Declared rather than derived, because the size of a model that has never + // been fetched anywhere is not knowable from here — and a wrong guess + // fails late, halfway through a download, which is the failure this exists + // to prevent. Zero means unconstrained, and a node that reports no storage + // is not excluded: unknown is not full. + MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"` + // NodeSelector restricts this profile to nodes carrying every one of these // labels, with these values. // @@ -344,6 +358,7 @@ func (m Model) PlacementProfiles() []placement.Profile { Priority: p.Priority, NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, + MinDiskGB: p.MinDiskGB, // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, diff --git a/internal/control/control.go b/internal/control/control.go index f6a67d8..77ef90b 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -75,6 +75,7 @@ type Controller struct { inv *inventory.Store cat *catalog.Live rec *reconciler.Reconciler + disp reconciler.Dispatcher sink EndpointSink usage UsageSource @@ -101,6 +102,7 @@ func New(cfg Config, cat *catalog.Live, disp reconciler.Dispatcher, sink Endpoin inv: inv, cat: cat, rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log), + disp: disp, sink: sink, stop: make(chan struct{}), done: make(chan struct{}), @@ -170,6 +172,21 @@ func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec } // Catalog exposes the model catalog. func (c *Controller) Catalog() *catalog.Catalog { return c.cat.Load() } +// WeightRemover can ask a node to delete cached model files. +// +// Separate from Dispatcher because it is not part of scheduling and not every +// dispatcher has it — the ones tests use do not. A caller gets nil rather than +// a method that silently does nothing. +type WeightRemover interface { + RemoveWeights(ctx context.Context, nodeID string, paths []string) error +} + +// WeightRemover returns the dispatcher's deletion capability, or nil. +func (c *Controller) WeightRemover() WeightRemover { + r, _ := c.disp.(WeightRemover) + return r +} + // Start runs the reconciler and begins publishing endpoints. func (c *Controller) Start(ctx context.Context) { c.rec.Start(ctx) diff --git a/internal/control/placement/disk_test.go b/internal/control/placement/disk_test.go new file mode 100644 index 0000000..84abf95 --- /dev/null +++ b/internal/control/placement/disk_test.go @@ -0,0 +1,93 @@ +package placement + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// withDisk gives a node a local filesystem of the stated size. +func withDisk(n inventory.Node, freeGB, totalGB int64) inventory.Node { + n.Storage = []discovery.Storage{{ + Path: "/var/lib/llmgw-models", + FreeBytes: freeGB << 30, + TotalBytes: totalGB << 30, + }} + return n +} + +// weightsProfile is a model that would have to be fetched: it names a source +// and says how much room that source needs. +func weightsProfile(source string, diskGB int) Profile { + return Profile{Name: "cpu", Engine: "llamacpp", Source: source, MinDiskGB: diskGB} +} + +func TestANodeWithoutRoomForTheWeightsIsNotACandidate(t *testing.T) { + // The failure this replaces: the node is picked, the agent starts + // downloading, and it dies partway through with a disk-full error that + // names neither the model nor the scheduler decision that caused it. + s := New(store(withDisk(node("full", "", 0), 2, 60))) + + res := s.Place([]Profile{weightsProfile("https://models/x.gguf", 20)}, nil) + if res.Placement != nil { + t.Fatalf("placed on a node with 2 GB free for a 20 GB model") + } + if !strings.Contains(res.Explain(), "needs 20") { + t.Errorf("the reason does not say what was needed:\n%s", res.Explain()) + } +} + +func TestRoomIsNotAskedForTwiceOnANodeThatAlreadyHasTheWeights(t *testing.T) { + // A node holding the file needs no space to hold it again. Without this, + // the one node best placed to run a model — because it already has it — is + // the one most likely to be rejected, since holding the file is what used + // the space up. + const source = "/var/lib/llmgw-models/big.gguf" + n := withDisk(node("holder", "", 0), 2, 60) + n.Weights = []discovery.LocalWeights{{Path: source, SizeBytes: 40 << 30}} + s := New(store(n)) + + res := s.Place([]Profile{weightsProfile(source, 20)}, nil) + if res.Placement == nil { + t.Fatalf("a node already holding the weights was rejected for lack of room:\n%s", res.Explain()) + } +} + +func TestANodeThatReportsNoDiskIsStillACandidate(t *testing.T) { + // An agent too old to report storage, and a node with no model directories, + // both report none. Excluding them would mean a control plane upgrade + // quietly empties every node in a fleet that has not upgraded yet — a worse + // failure than the one this check prevents. + s := New(store(node("silent", "", 0))) + + res := s.Place([]Profile{weightsProfile("https://models/x.gguf", 20)}, nil) + if res.Placement == nil { + t.Fatalf("a node that reported no storage was treated as a full one:\n%s", res.Explain()) + } +} + +func TestWithoutADeclaredSizeDiskIsNotConsidered(t *testing.T) { + // Every profile that existed before this field has no number, and must + // keep being placed exactly as it was. + s := New(store(withDisk(node("full", "", 0), 0, 60))) + + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", Source: "https://models/x.gguf"}}, nil) + if res.Placement == nil { + t.Fatalf("a profile with no declared size was rejected on disk:\n%s", res.Explain()) + } +} + +func TestSpaceOnASharedFilerDoesNotCountAsRoom(t *testing.T) { + // The node cannot spend it: it belongs to everyone mounting the filer. A + // node whose only storage is shared therefore reports no local disk, which + // lands in the same place as reporting nothing — unknown, so not excluded. + n := node("filer-only", "", 0) + n.Storage = []discovery.Storage{{ + Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true, + }} + if free, total := n.LocalDisk(); free != 0 || total != 0 { + t.Fatalf("shared space counted as local: %d/%d", free, total) + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 85521c2..951381e 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -36,6 +36,9 @@ type Profile struct { NodeSelector map[string]string // MinMemoryGB is the host RAM this profile needs. Zero is unconstrained. MinMemoryGB int + // MinDiskGB is the free space needed to hold the weights, checked only on + // nodes that do not already have them. Zero is unconstrained. + MinDiskGB int // Source is the model's weights, as the engine would be given it. Used to // prefer nodes that already hold them; empty asks for no preference. Source string @@ -401,6 +404,22 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje continue } } + // Room for the weights, and only for a node that would have to fetch + // them. A node already holding the file needs no space for it. + // + // Same two exemptions as memory, for the same reason: zero means the + // operator declared nothing, and a node reporting no storage is an + // older agent or one with no model directories, neither of which is a + // full disk. Without that second exemption a control plane upgrade + // would empty every node in a fleet that had not upgraded yet. + if p.MinDiskGB > 0 && p.Source != "" && !n.HasWeights(p.Source) { + if free, total := n.LocalDisk(); total > 0 { + if have := int(free / (1 << 30)); have < p.MinDiskGB { + reject(fmt.Sprintf("has %d GB free for weights, needs %d", have, p.MinDiskGB)) + continue + } + } + } if p.CPUOnly() { // Every ready node can run it; there is no card to be free.