Deleting unused model files says what went, and sets go whole

A sweep counted every file the nodes were asked to delete as freed as soon
as it asked, and a node refused a set's directory as not empty, so tens of
gigabytes were reported freed and nothing was. The agent removes a set's
directory whole (a link only as a link), and the control plane waits for
each node to list its files again, saying what it no longer lists and what
it kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 10:29:16 +01:00
1 parent 3bb52742a4
commit cba4897987
8 files changed
+293 -46

No files matched your search

+8 -9
View File
@@ -216,7 +216,7 @@ func (a *API) purgeFromNodes(ctx context.Context, sources []string, others []cat
names[filepath.Base(s)] = true
}
wanted := inUse(others)
var out []CollectResult
var rs []removal
for _, n := range a.ctl.Inventory().Nodes() {
size := map[string]int64{}
have := make([]string, 0, len(n.Weights))
@@ -235,15 +235,14 @@ func (a *API) purgeFromNodes(ctx context.Context, sources []string, others []cat
if len(paths) == 0 {
continue
}
res := CollectResult{Node: n.ID, Files: len(paths)}
for _, p := range paths {
res.Bytes += size[p]
rs = append(rs, removal{node: n.ID, paths: paths, size: size})
}
out := a.removeConfirmed(ctx, rm, rs)
for _, r := range out {
if r.Error != "" || r.Left > 0 {
slog.Warn("a purged model's copies were not all deleted on a node", "node", r.Node,
"left", r.Left, "err", r.Error)
}
if err := rm.RemoveWeights(ctx, n.ID, paths); err != nil {
res.Error, res.Files, res.Bytes = err.Error(), 0, 0
slog.Warn("a purged model's copies could not be deleted on a node", "node", n.ID, "err", err)
}
out = append(out, res)
}
return out
}
+110 -13
View File
@@ -1,11 +1,14 @@
package adminapi
import (
"context"
"encoding/json"
"net/http"
"sort"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
)
// OrphanView is cached weights on one node that no catalog model accounts for.
@@ -71,12 +74,100 @@ type CollectRequest struct {
Node string `json:"node,omitempty"`
}
// CollectResult is what was deleted on one node, or why nothing was.
// CollectResult is what was deleted on one node, or why nothing was: what
// the node no longer lists once asked, not what it was asked to delete.
type CollectResult struct {
Node string `json:"node"`
Files int `json:"files"`
Bytes int64 `json:"bytes"`
Error string `json:"error,omitempty"`
// Left and LeftBytes are what the node still lists: refused, or not
// gone by the time the sweep stopped waiting. Its agent's log says why.
Left int `json:"left,omitempty"`
LeftBytes int64 `json:"left_bytes,omitempty"`
Error string `json:"error,omitempty"`
}
// confirmWait is how long a sweep waits for the nodes to list their files
// without the ones deleted: they scan theirs every heartbeat, five seconds
// by default, and a large set takes a moment to delete.
var confirmWait = 30 * time.Second
// removal is what one node is asked to delete, with each file's size.
type removal struct {
node string
paths []string
size map[string]int64
}
// removeConfirmed asks each node to delete its paths and says what each then
// no longer lists. Counted as asked, a sweep said "25 GB freed" of files the
// node had refused and kept.
func (a *API) removeConfirmed(ctx context.Context, rm control.WeightRemover, rs []removal) []CollectResult {
out := make([]CollectResult, len(rs))
pending := map[int]bool{}
for i, r := range rs {
out[i].Node = r.node
if err := rm.RemoveWeights(ctx, r.node, r.paths); err != nil {
out[i].Error = err.Error()
for _, p := range r.paths {
out[i].Left++
out[i].LeftBytes += r.size[p]
}
continue
}
pending[i] = true
}
deadline := time.Now().Add(confirmWait)
for {
for i := range pending {
if left := a.stillListed(rs[i]); len(left) == 0 {
delete(pending, i)
}
}
if len(pending) == 0 || !time.Now().Before(deadline) {
break
}
select {
case <-ctx.Done():
deadline = time.Now()
case <-time.After(time.Second):
}
}
for i, r := range rs {
if out[i].Error != "" {
continue
}
left := a.stillListed(r)
for _, p := range r.paths {
if left[p] {
out[i].Left++
out[i].LeftBytes += r.size[p]
} else {
out[i].Files++
out[i].Bytes += r.size[p]
}
}
}
return out
}
// stillListed is which of a removal's paths its node still lists.
func (a *API) stillListed(r removal) map[string]bool {
left := map[string]bool{}
n, ok := a.ctl.Inventory().Get(r.node)
if !ok {
return left
}
want := map[string]bool{}
for _, p := range r.paths {
want[p] = true
}
for _, w := range n.Weights {
if want[w.Path] {
left[w.Path] = true
}
}
return left
}
// collectWeights deletes them.
@@ -97,20 +188,26 @@ func (a *API) collectWeights(w http.ResponseWriter, r *http.Request) {
// An empty body is the "all nodes" case and must not be an error.
_ = json.NewDecoder(r.Body).Decode(&body)
out := make([]CollectResult, 0)
// A node that has gone since the list was built is reported rather than
// failing the whole sweep: the other nodes' space is still worth
// freeing, and this one will be offered again next time.
var rs []removal
for _, o := range a.Orphans() {
if body.Node != "" && o.Node != body.Node {
continue
}
res := CollectResult{Node: o.Node, Files: len(o.Paths), Bytes: o.Bytes}
// A node that has gone since the list was built is reported rather than
// failing the whole sweep: the other nodes' space is still worth
// freeing, and this one will be offered again next time.
if err := rm.RemoveWeights(r.Context(), o.Node, o.Paths); err != nil {
res.Error = err.Error()
res.Files, res.Bytes = 0, 0
}
out = append(out, res)
rs = append(rs, removal{node: o.Node, paths: o.Paths, size: a.sizesOn(o.Node)})
}
writeJSON(w, http.StatusOK, out)
writeJSON(w, http.StatusOK, a.removeConfirmed(r.Context(), rm, rs))
}
// sizesOn is each file a node lists, by path, with its size.
func (a *API) sizesOn(node string) map[string]int64 {
out := map[string]int64{}
if n, ok := a.ctl.Inventory().Get(node); ok {
for _, w := range n.Weights {
out[w.Path] = w.SizeBytes
}
}
return out
}
+71
View File
@@ -0,0 +1,71 @@
package adminapi
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// deletingDispatcher deletes what it is asked to as a node would, by its
// files dropping out of what the node lists — but for refused, which it
// keeps, as an agent keeps a file it cannot prove it fetched.
type deletingDispatcher struct {
nullDispatcher
inv *inventory.Store
refused string
}
func (d *deletingDispatcher) RemoveWeights(_ context.Context, node string, paths []string) error {
n, _ := d.inv.Get(node)
n.Weights = slices.DeleteFunc(n.Weights, func(w discovery.LocalWeights) bool {
return w.Path != d.refused && slices.Contains(paths, w.Path)
})
d.inv.Upsert(n)
return nil
}
// What a sweep says it freed is what the node no longer lists: a file it
// kept is said to be left, not counted as freed.
func TestASweepSaysWhatWasDeletedNotWhatWasAsked(t *testing.T) {
confirmWait = 1500 * time.Millisecond
t.Cleanup(func() { confirmWait = 30 * time.Second })
gone := weights.CachePathFor("/var/lib/llmgw-weights", "http://mirror/old/v1/old.gguf")
set := weights.CachePathFor("/var/lib/llmgw-weights", "http://mirror/big/v1")
kept := weights.CachePathFor("/var/lib/llmgw-weights", "http://mirror/stuck/v1/stuck.gguf")
cat, _ := catalog.New(nil)
d := &deletingDispatcher{refused: kept}
ctl := control.New(control.DefaultConfig(), catalog.NewLive(cat), d, nullSink{}, slog.New(slog.DiscardHandler))
d.inv = ctl.Inventory()
node := gpuNode("n1", 1)
node.Weights = []discovery.LocalWeights{{Path: gone, SizeBytes: 1 << 30}, {Path: set, SizeBytes: 20 << 30},
{Path: kept, SizeBytes: 4 << 30}}
ctl.Inventory().Upsert(node)
a := New(ctl)
rec := httptest.NewRecorder()
a.collectWeights(rec, httptest.NewRequest(http.MethodPost, "/api/v1/weights/collect", strings.NewReader(`{"node":"n1"}`)))
var got []CollectResult
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatalf("%v: %s", err, rec.Body.String())
}
if len(got) != 1 {
t.Fatalf("results %+v", got)
}
r := got[0]
if r.Files != 2 || r.Bytes != 21<<30 || r.Left != 1 || r.LeftBytes != 4<<30 {
t.Errorf("result %+v: want 2 files and 21 GB freed, 1 file of 4 GB left", r)
}
}