diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d6d8ecf1..3ff8981d 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -201,6 +201,7 @@ type ControlMessage struct { // *ControlMessage_Stop // *ControlMessage_Drain // *ControlMessage_Ack + // *ControlMessage_RemoveWeights Msg isControlMessage_Msg `protobuf_oneof:"msg"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -279,6 +280,15 @@ func (x *ControlMessage) GetAck() *Ack { return nil } +func (x *ControlMessage) GetRemoveWeights() *RemoveWeights { + if x != nil { + if x, ok := x.Msg.(*ControlMessage_RemoveWeights); ok { + return x.RemoveWeights + } + } + return nil +} + type isControlMessage_Msg interface { isControlMessage_Msg() } @@ -299,6 +309,10 @@ type ControlMessage_Ack struct { Ack *Ack `protobuf:"bytes,4,opt,name=ack,proto3,oneof"` } +type ControlMessage_RemoveWeights struct { + RemoveWeights *RemoveWeights `protobuf:"bytes,5,opt,name=remove_weights,json=removeWeights,proto3,oneof"` +} + func (*ControlMessage_Start) isControlMessage_Msg() {} func (*ControlMessage_Stop) isControlMessage_Msg() {} @@ -307,6 +321,8 @@ func (*ControlMessage_Drain) isControlMessage_Msg() {} func (*ControlMessage_Ack) isControlMessage_Msg() {} +func (*ControlMessage_RemoveWeights) isControlMessage_Msg() {} + // Register is the first message on every stream. It carries the node's full // hardware inventory, so the control plane needs no separate discovery call and // a reconnecting agent automatically refreshes what the scheduler knows. @@ -1446,6 +1462,57 @@ func (x *EngineProfile) GetServedModelName() string { return "" } +// RemoveWeights deletes cached model files a node no longer needs. +// +// The paths are computed by the control plane from the catalog, but the agent +// checks them again before unlinking anything: it deletes only files inside its +// own cache directory whose names this system generated. A control plane that +// is wrong, compromised or simply newer than the agent must not be able to +// talk it into deleting a model an operator spent a week copying there. +type RemoveWeights struct { + state protoimpl.MessageState `protogen:"open.v1"` + Paths []string `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RemoveWeights) Reset() { + *x = RemoveWeights{} + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RemoveWeights) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveWeights) ProtoMessage() {} + +func (x *RemoveWeights) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. +func (*RemoveWeights) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} +} + +func (x *RemoveWeights) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + // StopShard terminates a shard and frees its GPUs. type StopShard struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1456,7 +1523,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1468,7 +1535,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1481,7 +1548,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *StopShard) GetShardId() string { @@ -1501,7 +1568,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1513,7 +1580,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1526,7 +1593,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *DrainShard) GetShardId() string { @@ -1548,7 +1615,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1560,7 +1627,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1573,7 +1640,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *Ack) GetMessageId() string { @@ -1606,12 +1673,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" + "\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" + "\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shardB\x05\n" + - "\x03msg\"\xc1\x01\n" + + "\x03msg\"\x83\x02\n" + "\x0eControlMessage\x12,\n" + "\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + - "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" + + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ack\x12@\n" + + "\x0eremove_weights\x18\x05 \x01(\v2\x17.llmgw.v1.RemoveWeightsH\x00R\rremoveWeightsB\x05\n" + "\x03msg\"\xa7\x05\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + @@ -1748,7 +1816,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"&\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" + + "\rRemoveWeights\x12\x14\n" + + "\x05paths\x18\x01 \x03(\tR\x05paths\"&\n" + "\tStopShard\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\"'\n" + "\n" + @@ -1777,7 +1847,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -1792,39 +1862,41 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus (*StartShard)(nil), // 11: llmgw.v1.StartShard (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile - (*StopShard)(nil), // 13: llmgw.v1.StopShard - (*DrainShard)(nil), // 14: llmgw.v1.DrainShard - (*Ack)(nil), // 15: llmgw.v1.Ack - nil, // 16: llmgw.v1.Register.LabelsEntry - nil, // 17: llmgw.v1.Register.EngineVersionsEntry - nil, // 18: llmgw.v1.EngineProfile.EnvEntry + (*RemoveWeights)(nil), // 13: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 14: llmgw.v1.StopShard + (*DrainShard)(nil), // 15: llmgw.v1.DrainShard + (*Ack)(nil), // 16: llmgw.v1.Ack + nil, // 17: llmgw.v1.Register.LabelsEntry + nil, // 18: llmgw.v1.Register.EngineVersionsEntry + nil, // 19: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 13, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 14, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 15, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 16, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 6, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 7, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 17, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 5, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 8, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 4, // 13: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage - 0, // 14: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 0, // 15: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 12, // 16: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 18, // 17: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 18: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 19: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 19, // [19:20] is the sub-list for method output_type - 18, // [18:19] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 18, // [18:18] is the sub-list for extension extendee - 0, // [0:18] is the sub-list for field type_name + 14, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 15, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 16, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 13, // 7: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 17, // 8: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 6, // 9: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 7, // 10: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 18, // 11: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 5, // 12: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 0, // 16: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 12, // 17: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 19, // 18: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 19: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 20: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 20, // [20:21] is the sub-list for method output_type + 19, // [19:20] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1842,6 +1914,7 @@ func file_llmgw_v1_agent_proto_init() { (*ControlMessage_Stop)(nil), (*ControlMessage_Drain)(nil), (*ControlMessage_Ack)(nil), + (*ControlMessage_RemoveWeights)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -1849,7 +1922,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 18, + NumMessages: 19, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 8fc6c2f7..f4792bf9 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -33,6 +33,7 @@ message ControlMessage { StopShard stop = 2; DrainShard drain = 3; Ack ack = 4; + RemoveWeights remove_weights = 5; } } @@ -266,6 +267,17 @@ message EngineProfile { string served_model_name = 11; } +// RemoveWeights deletes cached model files a node no longer needs. +// +// The paths are computed by the control plane from the catalog, but the agent +// checks them again before unlinking anything: it deletes only files inside its +// own cache directory whose names this system generated. A control plane that +// is wrong, compromised or simply newer than the agent must not be able to +// talk it into deleting a model an operator spent a week copying there. +message RemoveWeights { + repeated string paths = 1; +} + // StopShard terminates a shard and frees its GPUs. message StopShard { string shard_id = 1; diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 458139d5..c5f36d2c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -11,6 +11,8 @@ import ( "math/rand/v2" "net" "net/http" + "os" + "path/filepath" "strings" "sync" "time" @@ -581,6 +583,60 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}} } +// removeWeights deletes cached model files the control plane says are no +// longer needed. +// +// Every path is checked here as well as there, and that duplication is the +// point. The control plane computes these from the catalog, but it is the node +// that holds the files, and a node must not be talkable into deleting a model +// an operator spent a week copying onto it — not by a control plane that is +// wrong, not by one that is newer than this agent, not by one that has been +// tampered with. So this deletes only what it can prove llmgw itself wrote: +// inside its own cache directory, under a name this system generates. +// +// Failures are logged and skipped rather than reported upward. A file that is +// already gone is the expected outcome of a retry, and one that will not +// delete is a disk to look at, not a reason to fail a command whose purpose +// was to free space. +func (a *Agent) removeWeights(paths []string) { + cache := a.cfg.WeightsCache + if cache == "" && len(a.cfg.ModelDirs) > 0 { + cache = a.cfg.ModelDirs[0] + } + if cache == "" { + a.log.Warn("asked to remove weights with no cache directory configured") + return + } + cacheAbs, err := filepath.Abs(cache) + if err != nil { + a.log.Warn("cannot resolve the cache directory", "dir", cache, "err", err) + return + } + + for _, p := range paths { + abs, err := filepath.Abs(p) + if err != nil { + continue + } + // Directly inside the cache directory, not merely under it: a nested + // path would let ".." or a symlinked subdirectory reach elsewhere, and + // nothing this package writes is nested anyway. + if filepath.Dir(abs) != cacheAbs { + a.log.Warn("refused to remove a file outside the weights cache", "path", p) + continue + } + if !weights.Owned(abs) { + a.log.Warn("refused to remove a file llmgw did not fetch", "path", p) + continue + } + if err := os.Remove(abs); err != nil && !os.IsNotExist(err) { + a.log.Warn("could not remove cached weights", "path", p, "err", err) + continue + } + a.log.Info("removed cached weights no model asks for", "path", p) + } +} + func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) { switch m := msg.Msg.(type) { case *llmgwv1.ControlMessage_Start: @@ -591,6 +647,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha if err := a.sup.Drain(m.Drain.GetShardId()); err != nil { a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err) } + case *llmgwv1.ControlMessage_RemoveWeights: + a.removeWeights(m.RemoveWeights.GetPaths()) case *llmgwv1.ControlMessage_Ack: if !m.Ack.GetOk() { a.log.Warn("control plane rejected a message", diff --git a/internal/agent/removeweights_test.go b/internal/agent/removeweights_test.go new file mode 100644 index 00000000..107a3dea --- /dev/null +++ b/internal/agent/removeweights_test.go @@ -0,0 +1,96 @@ +package agent + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" +) + +// agentWithCache is an agent that only knows where its weights live, which is +// all removeWeights needs. +func agentWithCache(t *testing.T, dir string) *Agent { + t.Helper() + return &Agent{ + cfg: Config{ModelDirs: []string{dir}}, + log: slog.New(slog.DiscardHandler), + } +} + +func write(t *testing.T, path string) string { + t.Helper() + if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestTheAgentRefusesToDeleteWhatItDidNotFetch(t *testing.T) { + // The control plane computes these paths, but the node holds the files. A + // control plane that is wrong, newer than this agent, or tampered with must + // not be able to talk it into deleting a model an operator spent a week + // copying here. So the agent checks again, on its own. + dir := t.TempDir() + a := agentWithCache(t, dir) + + operators := write(t, filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0.gguf")) + fetched := write(t, weights.CachePathFor(dir, "https://models.internal/gone.gguf")) + + a.removeWeights([]string{operators, fetched}) + + if _, err := os.Stat(operators); err != nil { + t.Errorf("a file llmgw never fetched was deleted: %v", err) + } + if _, err := os.Stat(fetched); !os.IsNotExist(err) { + t.Errorf("the fetched file was not removed: %v", err) + } +} + +func TestTheAgentRefusesPathsOutsideItsCache(t *testing.T) { + // Even with a name that looks like one of ours. The cache directory is the + // boundary, and traversal out of it is the attack this closes. + dir := t.TempDir() + elsewhere := t.TempDir() + a := agentWithCache(t, dir) + + // A correctly named cache file, in the wrong directory. + outside := write(t, weights.CachePathFor(elsewhere, "https://models.internal/x.gguf")) + // And the same thing reached by traversal from inside the cache. + traversal := filepath.Join(dir, "..", filepath.Base(elsewhere), + filepath.Base(outside)) + + a.removeWeights([]string{outside, traversal}) + + if _, err := os.Stat(outside); err != nil { + t.Errorf("a file outside the cache was deleted: %v", err) + } +} + +func TestRemovingAFileThatIsAlreadyGoneIsNotAFailure(t *testing.T) { + // The expected outcome of a retry. It must not stop the rest of the batch. + dir := t.TempDir() + a := agentWithCache(t, dir) + + missing := weights.CachePathFor(dir, "https://models.internal/never.gguf") + present := write(t, weights.CachePathFor(dir, "https://models.internal/here.gguf")) + + a.removeWeights([]string{missing, present}) + + if _, err := os.Stat(present); !os.IsNotExist(err) { + t.Errorf("a missing file stopped the batch: %v", err) + } +} + +func TestWithNoCacheConfiguredNothingIsDeleted(t *testing.T) { + dir := t.TempDir() + a := &Agent{cfg: Config{}, log: slog.New(slog.DiscardHandler)} + kept := write(t, weights.CachePathFor(dir, "https://models.internal/x.gguf")) + + a.removeWeights([]string{kept}) + + if _, err := os.Stat(kept); err != nil { + t.Errorf("an agent with no cache directory deleted something: %v", err) + } +} diff --git a/internal/agent/weights/fetch.go b/internal/agent/weights/fetch.go index 3d070170..09ea796c 100644 --- a/internal/agent/weights/fetch.go +++ b/internal/agent/weights/fetch.go @@ -15,8 +15,6 @@ package weights import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" "io" @@ -262,12 +260,7 @@ func (f *Fetcher) cachePath(source string) (string, error) { if f.CacheDir == "" { return "", fmt.Errorf("no cache directory configured, so %s cannot be fetched", source) } - sum := sha256.Sum256([]byte(source)) - name := filepath.Base(source) - if name == "" || name == "." || name == "/" { - name = "weights" - } - return filepath.Join(f.CacheDir, hex.EncodeToString(sum[:8])+"-"+name), nil + return CachePathFor(f.CacheDir, source), nil } // isPath reports whether a source names a file on this machine. diff --git a/internal/agent/weights/gc.go b/internal/agent/weights/gc.go new file mode 100644 index 00000000..9c64b5e8 --- /dev/null +++ b/internal/agent/weights/gc.go @@ -0,0 +1,99 @@ +package weights + +import ( + "crypto/sha256" + "encoding/hex" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// cacheNamePattern matches a filename this package created. +// +// The eight-byte hash of the source, hex encoded, then a dash, then the +// original filename — exactly what cachePath writes. It is the only mark of +// ownership there is, and it has to be exact: everything else in a model +// directory was put there by the operator, and deleting one of those files +// could cost a week of someone's bandwidth. +var cacheNamePattern = regexp.MustCompile(`^[0-9a-f]{16}-`) + +// CachePathFor is where a source would be cached in the given directory. +// +// Exported so the control plane can work out which files in a node's reported +// list are ones llmgw fetched for a model it still wants, without asking the +// node anything. The agent and the control plane must agree on this name +// exactly, which is why there is one function and not two. +func CachePathFor(dir, source string) string { + sum := sha256.Sum256([]byte(source)) + name := filepath.Base(source) + if name == "" || name == "." || name == "/" { + name = "weights" + } + return filepath.Join(dir, hex.EncodeToString(sum[:8])+"-"+name) +} + +// Owned reports whether a path is one this package created. +// +// The dividing line of the whole feature. A file without the cache name was +// put there by the operator — by scp, by a shared mount, by hand — and llmgw +// has no business deleting it however unreferenced it looks. A file with the +// cache name is one llmgw downloaded, and llmgw can take it back. +func Owned(path string) bool { + return cacheNamePattern.MatchString(filepath.Base(path)) +} + +// Orphans lists cached files that no wanted source accounts for. +// +// `have` is what a node reports on disk and `wanted` is every source in the +// catalog; both are passed in so this is a calculation rather than a scan, and +// so it can be run on the control plane against a node's reported inventory. +// +// Only files this package created are ever returned. A source that is not +// fetched — a plain path, or a model id an engine resolves itself — produces +// no cache name and so protects nothing and deletes nothing, which is correct: +// those files are not in the cache to begin with. +func Orphans(have []string, wanted []string) []string { + keep := map[string]struct{}{} + for _, src := range wanted { + if !isRemote(src) { + // Nothing was fetched for it, so nothing in the cache belongs to + // it. Computing a name here would reserve a file that does not + // exist and could never be an orphan anyway. + continue + } + // By base name rather than full path: the control plane does not know + // each node's cache directory, and the name is what identifies the + // file wherever the node keeps it. + keep[filepath.Base(CachePathFor("", src))] = struct{}{} + } + + var out []string + seen := map[string]struct{}{} + for _, path := range have { + base := filepath.Base(path) + if !Owned(base) { + continue + } + if _, wanted := keep[base]; wanted { + continue + } + if _, dup := seen[path]; dup { + continue + } + seen[path] = struct{}{} + out = append(out, path) + } + sort.Strings(out) + return out +} + +// PartialName reports whether a path is the temporary file a fetch writes to. +// +// Those are not orphans in the same sense — a fetch may be running right now — +// so they are named separately rather than swept up with the rest. A `.part` +// left behind by a killed agent is real garbage, but only its own node can +// tell whether it is stale, and nothing here can. +func PartialName(path string) bool { + return strings.HasSuffix(path, ".part") +} diff --git a/internal/agent/weights/gc_test.go b/internal/agent/weights/gc_test.go new file mode 100644 index 00000000..aaef840c --- /dev/null +++ b/internal/agent/weights/gc_test.go @@ -0,0 +1,116 @@ +package weights + +import ( + "path/filepath" + "slices" + "testing" +) + +func TestOnlyFilesLLMGWFetchedAreEverOrphans(t *testing.T) { + // The invariant the whole feature rests on. Everything without the cache + // name was put there by the operator — scp, a shared mount, a copy by hand + // — and deleting one could cost a week of someone's bandwidth. It does not + // matter how unreferenced it looks. + const dir = "/var/lib/llmgw-models" + have := []string{ + filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0-00001-of-00002.gguf"), + filepath.Join(dir, "some-model.safetensors"), + filepath.Join(dir, "notes.txt"), + CachePathFor(dir, "https://models.internal/gone.gguf"), + } + + got := Orphans(have, nil) + if len(got) != 1 { + t.Fatalf("expected only the fetched file, got %v", got) + } + if !Owned(got[0]) { + t.Errorf("returned a file llmgw did not fetch: %s", got[0]) + } +} + +func TestAFetchedFileStillInTheCatalogIsKept(t *testing.T) { + const dir = "/var/lib/llmgw-models" + const wanted = "hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf" + const gone = "https://models.internal/old.gguf" + + have := []string{CachePathFor(dir, wanted), CachePathFor(dir, gone)} + got := Orphans(have, []string{wanted}) + + if slices.Contains(got, CachePathFor(dir, wanted)) { + t.Error("a file the catalog still points at was called an orphan") + } + if !slices.Contains(got, CachePathFor(dir, gone)) { + t.Errorf("a file nothing points at was not found: %v", got) + } +} + +func TestTheCacheNameMatchesWhatAFetchActuallyWrites(t *testing.T) { + // The control plane proposes deletions by recomputing this name from the + // catalog. If it ever drifted from what a fetch writes, the proposal would + // name a file that is in use — so the two must be one function, and this + // is what says so. + f := &Fetcher{CacheDir: "/cache"} + const source = "https://models.internal/x.gguf" + + got, err := f.cachePath(source) + if err != nil { + t.Fatal(err) + } + if want := CachePathFor("/cache", source); got != want { + t.Errorf("a fetch writes %q but the catalog would look for %q", got, want) + } +} + +func TestOneOfTwoLookalikeFilesIsNotMistakenForTheOther(t *testing.T) { + // That the two cache to different names is covered where the naming lives. + // What matters here is the consequence: deleting the unreferenced one must + // not take the referenced one with it, and both end in "model.gguf". + a := CachePathFor("/cache", "https://one.example/model.gguf") + b := CachePathFor("/cache", "https://two.example/model.gguf") + + got := Orphans([]string{a, b}, []string{"https://one.example/model.gguf"}) + if !slices.Equal(got, []string{b}) { + t.Errorf("orphans = %v, want only %s", got, b) + } +} + +func TestASourceThatIsNeverFetchedProtectsNothingAndDeletesNothing(t *testing.T) { + // A plain path, or a model id an engine resolves itself, puts nothing in + // the cache. Computing a cache name for it would reserve a file that does + // not exist, and it must not stop a real orphan being found either. + const dir = "/models" + orphan := CachePathFor(dir, "https://models.internal/gone.gguf") + + got := Orphans([]string{orphan, filepath.Join(dir, "local.gguf")}, + []string{"/models/local.gguf", "Qwen/Qwen2.5-7B-Instruct"}) + + if !slices.Equal(got, []string{orphan}) { + t.Errorf("orphans = %v, want only %s", got, orphan) + } +} + +func TestAnEmptyCatalogDoesNotMakeOperatorFilesDisposable(t *testing.T) { + // The dangerous case: every model removed from the catalog at once. Files + // llmgw fetched become orphans, which is right. Files it did not must stay + // untouched, which is the whole point. + const dir = "/models" + have := []string{ + filepath.Join(dir, "hand-placed.gguf"), + CachePathFor(dir, "https://models.internal/a.gguf"), + } + got := Orphans(have, nil) + if len(got) != 1 || got[0] != CachePathFor(dir, "https://models.internal/a.gguf") { + t.Errorf("orphans = %v", got) + } +} + +func TestPartialsAreNotSweptUpWithTheRest(t *testing.T) { + // A fetch may be running right now, and only the node itself can tell a + // live download from one a killed agent left behind. + if !PartialName("/models/abc-model.gguf.part") { + t.Error("a .part file was not recognised") + } + if PartialName("/models/abc-model.gguf") { + t.Error("a finished file was called partial") + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index b98c888b..1a9e6373 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -126,6 +126,7 @@ func newCtlCmd() *cobra.Command { newModelCmd(client), newEngineCheckCmd(), newToolCheckCmd(), + newWeightsCmd(client, &asJSON), newCertCmd(), ) return cmd diff --git a/internal/cli/ctlweights.go b/internal/cli/ctlweights.go new file mode 100644 index 00000000..786e1241 --- /dev/null +++ b/internal/cli/ctlweights.go @@ -0,0 +1,153 @@ +package cli + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// newWeightsCmd reports and reclaims model files nothing needs. +// +// The dangerous half of storage management, so its shape is defensive: listing +// is the default and deleting takes a flag, only files llmgw fetched are ever +// candidates, and the node checks every path again before unlinking it. +// +// What it will never touch is the more important half of the contract. Weights +// an operator put on a node — by scp, by a shared mount, by hand — are none of +// this system's business however unreferenced they look, and a model that took +// a week to copy must not be removable by a command someone ran to tidy up. +func newWeightsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + cmd := &cobra.Command{ + Use: "weights", + Short: "Show cached model files no model in the catalog needs", + Long: "Lists weights llmgw downloaded for models that are no longer in the\n" + + "catalog, per node, with what deleting them would free.\n\n" + + "Only files llmgw fetched are considered. Anything an operator put on a\n" + + "node is left alone, whatever the catalog says — llmgw observes storage\n" + + "rather than owning it, and only takes back what it put there itself.\n\n" + + "Nothing is deleted without --collect.", + Args: cobra.NoArgs, + } + var collect bool + var node string + + cmd.Flags().BoolVar(&collect, "collect", false, "delete them, rather than only listing") + cmd.Flags().StringVar(&node, "node", "", "limit to one node") + + cmd.RunE = func(cmd *cobra.Command, _ []string) error { + if collect { + return collectOrphans(cmd, client(), node, *asJSON) + } + return listOrphanWeights(cmd, client(), node, *asJSON) + } + return cmd +} + +func listOrphanWeights(cmd *cobra.Command, c *ctlClient, node string, asJSON bool) error { + var found []adminapi.OrphanView + if err := c.get(cmd.Context(), "/api/v1/weights/orphans", &found); err != nil { + return err + } + if node != "" { + found = onlyNode(found, node) + } + if asJSON { + return emitJSON(found) + } + if len(found) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), + "Nothing to reclaim: every file llmgw fetched is still wanted by a model.") + return nil + } + + w := tab() + fmt.Fprintln(w, "NODE\tFILES\tWOULD FREE") + var total int64 + for _, o := range found { + fmt.Fprintf(w, "%s\t%d\t%s\n", o.Node, len(o.Paths), fileSize(o.Bytes)) + total += o.Bytes + } + _ = w.Flush() + + for _, o := range found { + for _, p := range o.Paths { + fmt.Fprintf(cmd.OutOrStdout(), "\n %s: %s", o.Node, p) + } + } + fmt.Fprintf(cmd.OutOrStdout(), "\n\n%s in total. Delete with --collect.\n", fileSize(total)) + return nil +} + +func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool) error { + body, err := json.Marshal(map[string]string{"node": node}) + if err != nil { + return err + } + var results []struct { + Node string `json:"node"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` + } + if err := c.post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil { + return err + } + if asJSON { + return emitJSON(results) + } + if len(results) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "Nothing to reclaim.") + return nil + } + var freed int64 + for _, r := range results { + if r.Error != "" { + // Reported rather than fatal: the other nodes' space is still worth + // freeing, and this one will be offered again next time. + fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", r.Node, r.Error) + continue + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: asked to remove %d files, %s\n", + r.Node, r.Files, fileSize(r.Bytes)) + freed += r.Bytes + } + // "asked", not "removed": each node decides, and one that refuses a path it + // cannot prove it fetched is the feature working rather than failing. + fmt.Fprintf(cmd.OutOrStdout(), "\n%s requested. The nodes verify each path before deleting it.\n", + fileSize(freed)) + return nil +} + +func onlyNode(in []adminapi.OrphanView, node string) []adminapi.OrphanView { + out := make([]adminapi.OrphanView, 0, len(in)) + for _, o := range in { + if o.Node == node { + out = append(out, o) + } + } + return out +} + +// fileSize renders a size the way someone reads one when deciding whether a +// file is worth deleting. +// +// Separate from the per-token cache size renderer in modelplan, which never +// leaves KiB and MiB. Model files are tens of gigabytes, and "8601.6 MiB" is +// not a number anyone weighs a decision against. +func fileSize(b int64) string { + switch { + case b >= 1<<30: + return fmt.Sprintf("%.1f GB", float64(b)/(1<<30)) + case b >= 1<<20: + return fmt.Sprintf("%.0f MB", float64(b)/(1<<20)) + case b > 0: + return fmt.Sprintf("%d B", b) + default: + // The node reported no size. Saying "0 B" would suggest deleting it + // gains nothing, which is a different claim from not knowing. + return "size unknown" + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 2b9792ac..6024ad5f 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -40,6 +40,10 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/nodes", a.listNodes) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) + // Listing is a read and deleting is a separate verb, so a mistyped + // command cannot remove anything. + mux.HandleFunc("GET /api/v1/weights/orphans", a.listOrphans) + mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights) mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement) mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState) } diff --git a/internal/control/adminapi/weights.go b/internal/control/adminapi/weights.go new file mode 100644 index 00000000..2f7927b8 --- /dev/null +++ b/internal/control/adminapi/weights.go @@ -0,0 +1,107 @@ +package adminapi + +import ( + "encoding/json" + "net/http" + "sort" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" +) + +// OrphanView is cached weights on one node that no catalog model accounts for. +type OrphanView struct { + Node string `json:"node"` + Paths []string `json:"paths"` + // Bytes is what deleting them would free, as far as the node reported. A + // file whose size was not reported counts as zero rather than being left + // out, so the list and the number always describe the same set. + Bytes int64 `json:"bytes"` +} + +// Orphans finds cached weights nothing in the catalog points at. +// +// Computed here from what nodes already report, so listing costs no round trip +// and a node that is offline simply contributes nothing — its files are not +// deletable while it is gone anyway. +// +// Only files llmgw fetched are ever considered. Everything else in a model +// directory was put there by the operator and is none of this system's +// business, however unreferenced it looks. +func (a *API) Orphans() []OrphanView { + var wanted []string + for _, m := range a.ctl.Catalog().Models() { + wanted = append(wanted, m.Source) + } + + out := make([]OrphanView, 0) + for _, n := range a.ctl.Inventory().Nodes() { + size := map[string]int64{} + have := make([]string, 0, len(n.Weights)) + for _, w := range n.Weights { + have = append(have, w.Path) + size[w.Path] = w.SizeBytes + } + + paths := weights.Orphans(have, wanted) + if len(paths) == 0 { + continue + } + v := OrphanView{Node: n.ID, Paths: paths} + for _, p := range paths { + v.Bytes += size[p] + } + out = append(out, v) + } + sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) + return out +} + +func (a *API) listOrphans(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.Orphans()) +} + +// collectWeights deletes them. +// +// A proposal, not an instruction: each node checks every path again before +// unlinking it, and refuses anything it cannot prove it fetched. The two +// checks are deliberate duplication — the control plane knows the catalog and +// the node knows its disk, and only the node can be sure. +func (a *API) collectWeights(w http.ResponseWriter, r *http.Request) { + rm := a.ctl.WeightRemover() + if rm == nil { + writeError(w, http.StatusNotImplemented, + "this control plane cannot ask nodes to delete files") + return + } + + var body struct { + // Node limits it to one node. Empty means every node, which is the + // common case after removing a model from the catalog. + Node string `json:"node,omitempty"` + } + // An empty body is the "all nodes" case and must not be an error. + _ = json.NewDecoder(r.Body).Decode(&body) + + type result struct { + Node string `json:"node"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` + } + out := make([]result, 0) + for _, o := range a.Orphans() { + if body.Node != "" && o.Node != body.Node { + continue + } + res := result{Node: o.Node, Files: len(o.Paths), Bytes: o.Bytes} + // A node that has gone since the list was built is reported rather than + // failing the whole sweep: the other nodes' space is still worth + // freeing, and this one will be offered again next time. + if err := rm.RemoveWeights(r.Context(), o.Node, o.Paths); err != nil { + res.Error = err.Error() + res.Files, res.Bytes = 0, 0 + } + out = append(out, res) + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index de2b7c5c..3309028a 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -340,6 +340,22 @@ func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error { }) } +// RemoveWeights asks a node to delete cached files no model needs. +// +// The node checks every path again before unlinking it, so this is a proposal +// rather than an instruction — which is the right shape for a message that +// deletes data the control plane cannot see. +func (s *Service) RemoveWeights(_ context.Context, nodeID string, paths []string) error { + if len(paths) == 0 { + return nil + } + return s.send(nodeID, &llmgwv1.ControlMessage{ + Msg: &llmgwv1.ControlMessage_RemoveWeights{ + RemoveWeights: &llmgwv1.RemoveWeights{Paths: paths}, + }, + }) +} + func (s *Service) send(nodeID string, msg *llmgwv1.ControlMessage) error { s.mu.RLock() c, ok := s.conns[nodeID] diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 7a17ce6c..644fbff7 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -71,6 +71,20 @@ type Profile struct { // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` + // MinDiskGB is the free space a node needs before it can hold these + // weights. + // + // Only checked when the node does not already have them: a node holding + // the file needs no room for it, and requiring the space twice would keep + // a model off the one node best placed to run it. + // + // Declared rather than derived, because the size of a model that has never + // been fetched anywhere is not knowable from here — and a wrong guess + // fails late, halfway through a download, which is the failure this exists + // to prevent. Zero means unconstrained, and a node that reports no storage + // is not excluded: unknown is not full. + MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"` + // NodeSelector restricts this profile to nodes carrying every one of these // labels, with these values. // @@ -344,6 +358,7 @@ func (m Model) PlacementProfiles() []placement.Profile { Priority: p.Priority, NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, + MinDiskGB: p.MinDiskGB, // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, diff --git a/internal/control/control.go b/internal/control/control.go index f6a67d87..77ef90b5 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -75,6 +75,7 @@ type Controller struct { inv *inventory.Store cat *catalog.Live rec *reconciler.Reconciler + disp reconciler.Dispatcher sink EndpointSink usage UsageSource @@ -101,6 +102,7 @@ func New(cfg Config, cat *catalog.Live, disp reconciler.Dispatcher, sink Endpoin inv: inv, cat: cat, rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log), + disp: disp, sink: sink, stop: make(chan struct{}), done: make(chan struct{}), @@ -170,6 +172,21 @@ func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec } // Catalog exposes the model catalog. func (c *Controller) Catalog() *catalog.Catalog { return c.cat.Load() } +// WeightRemover can ask a node to delete cached model files. +// +// Separate from Dispatcher because it is not part of scheduling and not every +// dispatcher has it — the ones tests use do not. A caller gets nil rather than +// a method that silently does nothing. +type WeightRemover interface { + RemoveWeights(ctx context.Context, nodeID string, paths []string) error +} + +// WeightRemover returns the dispatcher's deletion capability, or nil. +func (c *Controller) WeightRemover() WeightRemover { + r, _ := c.disp.(WeightRemover) + return r +} + // Start runs the reconciler and begins publishing endpoints. func (c *Controller) Start(ctx context.Context) { c.rec.Start(ctx) diff --git a/internal/control/placement/disk_test.go b/internal/control/placement/disk_test.go new file mode 100644 index 00000000..84abf950 --- /dev/null +++ b/internal/control/placement/disk_test.go @@ -0,0 +1,93 @@ +package placement + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// withDisk gives a node a local filesystem of the stated size. +func withDisk(n inventory.Node, freeGB, totalGB int64) inventory.Node { + n.Storage = []discovery.Storage{{ + Path: "/var/lib/llmgw-models", + FreeBytes: freeGB << 30, + TotalBytes: totalGB << 30, + }} + return n +} + +// weightsProfile is a model that would have to be fetched: it names a source +// and says how much room that source needs. +func weightsProfile(source string, diskGB int) Profile { + return Profile{Name: "cpu", Engine: "llamacpp", Source: source, MinDiskGB: diskGB} +} + +func TestANodeWithoutRoomForTheWeightsIsNotACandidate(t *testing.T) { + // The failure this replaces: the node is picked, the agent starts + // downloading, and it dies partway through with a disk-full error that + // names neither the model nor the scheduler decision that caused it. + s := New(store(withDisk(node("full", "", 0), 2, 60))) + + res := s.Place([]Profile{weightsProfile("https://models/x.gguf", 20)}, nil) + if res.Placement != nil { + t.Fatalf("placed on a node with 2 GB free for a 20 GB model") + } + if !strings.Contains(res.Explain(), "needs 20") { + t.Errorf("the reason does not say what was needed:\n%s", res.Explain()) + } +} + +func TestRoomIsNotAskedForTwiceOnANodeThatAlreadyHasTheWeights(t *testing.T) { + // A node holding the file needs no space to hold it again. Without this, + // the one node best placed to run a model — because it already has it — is + // the one most likely to be rejected, since holding the file is what used + // the space up. + const source = "/var/lib/llmgw-models/big.gguf" + n := withDisk(node("holder", "", 0), 2, 60) + n.Weights = []discovery.LocalWeights{{Path: source, SizeBytes: 40 << 30}} + s := New(store(n)) + + res := s.Place([]Profile{weightsProfile(source, 20)}, nil) + if res.Placement == nil { + t.Fatalf("a node already holding the weights was rejected for lack of room:\n%s", res.Explain()) + } +} + +func TestANodeThatReportsNoDiskIsStillACandidate(t *testing.T) { + // An agent too old to report storage, and a node with no model directories, + // both report none. Excluding them would mean a control plane upgrade + // quietly empties every node in a fleet that has not upgraded yet — a worse + // failure than the one this check prevents. + s := New(store(node("silent", "", 0))) + + res := s.Place([]Profile{weightsProfile("https://models/x.gguf", 20)}, nil) + if res.Placement == nil { + t.Fatalf("a node that reported no storage was treated as a full one:\n%s", res.Explain()) + } +} + +func TestWithoutADeclaredSizeDiskIsNotConsidered(t *testing.T) { + // Every profile that existed before this field has no number, and must + // keep being placed exactly as it was. + s := New(store(withDisk(node("full", "", 0), 0, 60))) + + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", Source: "https://models/x.gguf"}}, nil) + if res.Placement == nil { + t.Fatalf("a profile with no declared size was rejected on disk:\n%s", res.Explain()) + } +} + +func TestSpaceOnASharedFilerDoesNotCountAsRoom(t *testing.T) { + // The node cannot spend it: it belongs to everyone mounting the filer. A + // node whose only storage is shared therefore reports no local disk, which + // lands in the same place as reporting nothing — unknown, so not excluded. + n := node("filer-only", "", 0) + n.Storage = []discovery.Storage{{ + Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true, + }} + if free, total := n.LocalDisk(); free != 0 || total != 0 { + t.Fatalf("shared space counted as local: %d/%d", free, total) + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 85521c2f..951381ea 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -36,6 +36,9 @@ type Profile struct { NodeSelector map[string]string // MinMemoryGB is the host RAM this profile needs. Zero is unconstrained. MinMemoryGB int + // MinDiskGB is the free space needed to hold the weights, checked only on + // nodes that do not already have them. Zero is unconstrained. + MinDiskGB int // Source is the model's weights, as the engine would be given it. Used to // prefer nodes that already hold them; empty asks for no preference. Source string @@ -401,6 +404,22 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje continue } } + // Room for the weights, and only for a node that would have to fetch + // them. A node already holding the file needs no space for it. + // + // Same two exemptions as memory, for the same reason: zero means the + // operator declared nothing, and a node reporting no storage is an + // older agent or one with no model directories, neither of which is a + // full disk. Without that second exemption a control plane upgrade + // would empty every node in a fleet that had not upgraded yet. + if p.MinDiskGB > 0 && p.Source != "" && !n.HasWeights(p.Source) { + if free, total := n.LocalDisk(); total > 0 { + if have := int(free / (1 << 30)); have < p.MinDiskGB { + reject(fmt.Sprintf("has %d GB free for weights, needs %d", have, p.MinDiskGB)) + continue + } + } + } if p.CPUOnly() { // Every ready node can run it; there is no card to be free.